From 96135b4428ed4fcd436ac88f4562143be51422ff Mon Sep 17 00:00:00 2001 From: aidenvaines-bjss Date: Thu, 21 Aug 2025 09:40:48 +0100 Subject: [PATCH] CCM-5833 Adding ServiceDiscovery Dashboard --- .github/workflows/scorecard.yml | 4 +- .../obs/iam_role_grafana_workspace.tf | 19 +- .../overview/deployed-versions.json | 457 ++++++++++++++++++ .../obsconfig/grafana_data_source_athena.tf | 13 + scripts/githooks/check-todos.sh | 4 +- 5 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 infrastructure/terraform/components/obsconfig/dashboards/overview/deployed-versions.json create mode 100644 infrastructure/terraform/components/obsconfig/grafana_data_source_athena.tf diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5552785d..df4947e5 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,7 +37,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif @@ -68,6 +68,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@fca7ace96b7d713c7035871441bd52efbe39e27e # v3.28.19 + uses: github/codeql-action/upload-sarif@181d5eefc20863364f96762470ba6f862bdef56b # v3.29.2 with: sarif_file: results.sarif diff --git a/infrastructure/terraform/components/obs/iam_role_grafana_workspace.tf b/infrastructure/terraform/components/obs/iam_role_grafana_workspace.tf index 7237c744..82ec741e 100644 --- a/infrastructure/terraform/components/obs/iam_role_grafana_workspace.tf +++ b/infrastructure/terraform/components/obs/iam_role_grafana_workspace.tf @@ -54,9 +54,26 @@ data "aws_iam_policy_document" "grafana_cross_account_access" { "arn:aws:iam::${statement.value.account_id}:role/${replace(local.csi, "nhs", "nhs-notify")}-cross-access-role", ] } - } + statement { + sid = "AllowAthenaWorkspaceAccess" + effect = "Allow" + actions = [ + "s3:AbortMultipartUpload", + "s3:CreateBucket", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutBucketPublicAccessBlock", + "s3:PutObject", + ] + resources = [ + "${local.acct.s3_buckets["observability"]["arn"]}/athena-output/*" + ] + } } resource "aws_iam_policy" "grafana_cross_account_access" { diff --git a/infrastructure/terraform/components/obsconfig/dashboards/overview/deployed-versions.json b/infrastructure/terraform/components/obsconfig/dashboards/overview/deployed-versions.json new file mode 100644 index 00000000..8788f15f --- /dev/null +++ b/infrastructure/terraform/components/obsconfig/dashboards/overview/deployed-versions.json @@ -0,0 +1,457 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 68, + "links": [], + "liveNow": true, + "panels": [ + { + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Static Environment Deployments", + "mode": "markdown" + }, + "pluginVersion": "10.4.1", + "type": "text" + }, + { + "datasource": { + "type": "grafana-athena-datasource", + "uid": "fevkyppe14934d" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "minWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "component" + }, + "properties": [ + { + "id": "custom.width", + "value": 113 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "group" + }, + "properties": [ + { + "id": "custom.width", + "value": 205 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version_bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 231 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version_internal" + }, + "properties": [ + { + "id": "custom.width", + "value": 151 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "environment" + }, + "properties": [ + { + "id": "custom.width", + "value": 131 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "commit_id_internal" + }, + "properties": [ + { + "id": "custom.width", + "value": 378 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "commit_id_bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 277 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deployed_at" + }, + "properties": [ + { + "id": "custom.width", + "value": 221 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 164 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "connectionArgs": { + "catalog": "__default", + "database": "nhs-main-acct", + "region": "__default", + "resultReuseEnabled": false, + "resultReuseMaxAgeInMinutes": 60 + }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "fevkyppe14934d" + }, + "format": 1, + "rawSQL": "SELECT bounded_context, \"group\", environment, component, version_internal, version_bounded_context, commit_id_internal, commit_id_bounded_context, deployed_at\nFROM (\n SELECT\n t.*,\n ROW_NUMBER() OVER (\n PARTITION BY t.bounded_context, t.component, t.\"group\"\n ORDER BY t.deployed_at DESC\n ) AS rn\n FROM \"nhs-main-acct-deployments\" t\n WHERE t.environment = 'main'\n) sub\nWHERE sub.rn = 1;", + "refId": "A", + "table": "nhs-main-acct-deployments" + } + ], + "type": "table" + }, + { + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 4, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Dynamic Environment Deployments", + "mode": "markdown" + }, + "pluginVersion": "10.4.1", + "type": "text" + }, + { + "datasource": { + "type": "grafana-athena-datasource", + "uid": "fevkyppe14934d" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "minWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "component" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "group" + }, + "properties": [ + { + "id": "custom.width", + "value": 217 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version_bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 359 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "version_internal" + }, + "properties": [ + { + "id": "custom.width", + "value": 315 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "commit_id_bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 295 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "commit_id_internal" + }, + "properties": [ + { + "id": "custom.width", + "value": 419 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deployed_at" + }, + "properties": [ + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "bounded_context" + }, + "properties": [ + { + "id": "custom.width", + "value": 187 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "environment" + }, + "properties": [ + { + "id": "custom.width", + "value": 122 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "connectionArgs": { + "catalog": "__default", + "database": "nhs-main-acct", + "region": "__default", + "resultReuseEnabled": false, + "resultReuseMaxAgeInMinutes": 60 + }, + "datasource": { + "type": "grafana-athena-datasource", + "uid": "fevkyppe14934d" + }, + "format": 1, + "rawSQL": "SELECT bounded_context, \"group\", environment, component, version_internal, version_bounded_context, commit_id_internal, commit_id_bounded_context, deployed_at\nFROM (\n SELECT\n t.*,\n ROW_NUMBER() OVER (\n PARTITION BY t.bounded_context, t.component, t.\"group\"\n ORDER BY t.deployed_at DESC\n ) AS rn\n FROM \"nhs-main-acct-deployments\" t\n WHERE t.environment != 'main'\n ORDER BY version_bounded_context\n) sub\nWHERE sub.rn = 1;", + "refId": "A", + "table": "nhs-main-acct-deployments" + } + ], + "type": "table" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "Deployed Component Versions", + "weekStart": "" +} diff --git a/infrastructure/terraform/components/obsconfig/grafana_data_source_athena.tf b/infrastructure/terraform/components/obsconfig/grafana_data_source_athena.tf new file mode 100644 index 00000000..bd3a4f6e --- /dev/null +++ b/infrastructure/terraform/components/obsconfig/grafana_data_source_athena.tf @@ -0,0 +1,13 @@ +resource "grafana_data_source" "athena_deployments" { + type = "grafana-athena-datasource" + name = "${local.csi}-athena-deployments" + + json_data_encoded = jsonencode({ + defaultRegion = "eu-west-2" + authType = "ec2_iam_role" + catalog = "AwsDataCatalog" + database = local.acct.glue_database_name + workgroup = local.acct.athena_workgroup["name"] + outputLocation = local.acct.athena_workgroup["output_location"] + }) +} diff --git a/scripts/githooks/check-todos.sh b/scripts/githooks/check-todos.sh index 83b7a80e..4135cb2a 100755 --- a/scripts/githooks/check-todos.sh +++ b/scripts/githooks/check-todos.sh @@ -120,7 +120,7 @@ function search_todos() { # If the file is excluded, skip it if [ "$skip" = false ] && [ -f "$file" ]; then - file_todos=$(grep -nHiE '\bTODO\b' "$file" || true) + file_todos=$(grep -nHiE '\bTODO(:| )' "$file" || true) [ -n "$file_todos" ] && todos+="$file_todos\n" fi done @@ -136,7 +136,7 @@ function filter_todos_with_valid_jira_ticket() { while IFS= read -r line; do # Only lines with TODO but without a valid JIRA ticket - if grep -qnHiE '\bTODO\b' <<< "$line"; then + if grep -qnHiE '\bTODO(:| )' <<< "$line"; then if ! [[ "$line" =~ $jira_regex ]]; then todos_without_ticket+="$line\n" fi