From 239df8be665fcd4342db255b37d48ea43861b3d4 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Wed, 2 Apr 2025 15:36:35 +0100 Subject: [PATCH 01/11] Enabling use of podman --- Makefile | 3 +- build/automation/lib/docker.mk | 97 ++++++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 2063bbe5c..a69d381d8 100644 --- a/Makefile +++ b/Makefile @@ -335,7 +335,8 @@ batch-delete-ecr-images: # Mandatory - LIST_OF_DIGESTS: [list of "sha:digest" se tester-build: ### Build tester docker image cat $(APPLICATION_DIR)/*/requirements.txt $(APPLICATION_DIR)/requirements-dev.txt | sort --unique > $(DOCKER_DIR)/tester/assets/requirements.txt - make -s docker-image NAME=tester + # make -s docker-image NAME=tester + make -s podman-image NAME=tester tester-clean: rm -fv $(DOCKER_DIR)/tester/assets/*.txt diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index cd6466beb..8fb725cf4 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -67,31 +67,33 @@ docker-create-from-template: ### Create Docker image from template - mandatory: docker-config: ### Configure Docker networking docker network create $(DOCKER_NETWORK) 2> /dev/null ||: -docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: VERSION,FROM_CACHE=true,BUILD_OPTS=[build options],EXAMPLE=true +######### Trial for podman ############# + +podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: VERSION, FROM_CACHE=true, BUILD_OPTS=[build options], EXAMPLE=true reg=$$(make _docker-get-reg) # Try to execute `make build` from the image directory - if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) - make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) + make build __PODMAN_BUILD=true PODMAN_REGISTRY=$(PODMAN_LIBRARY_REGISTRY) exit - elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then cd $(DOCKER_CUSTOM_DIR)/$(NAME) - make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) - elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) + elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then cd $(DOCKER_DIR)/$(NAME) - make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) - elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) + elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) - make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) + make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) fi - # Dockerfile + # Dockerfile (now Podman) make NAME=$(NAME) \ docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ docker-image-set-version VERSION=$(VERSION) # Cache cache_from= if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - make docker-pull NAME=$(NAME) VERSION=latest + make podman-pull NAME=$(NAME) VERSION=latest cache_from="--cache-from $$reg/$(NAME):latest" fi # Build @@ -99,7 +101,7 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) export VERSION=$$(make docker-image-get-version) make -s file-replace-variables FILE=$$dir/Dockerfile.effective - docker buildx build --rm \ + podman build --rm \ --build-arg IMAGE=$$IMAGE \ --build-arg VERSION=$$VERSION \ --build-arg BUILD_ID=$(BUILD_ID) \ @@ -108,7 +110,6 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ - --output type=docker \ --label name=$$IMAGE \ --label version=$$VERSION \ --label build-id=$(BUILD_ID) \ @@ -122,12 +123,76 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ $$dir # Tag - docker tag \ + podman tag \ $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest - docker rmi --force $$(docker images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: + podman rmi --force $$(podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: make docker-image-keep-latest-only NAME=$(NAME) - docker image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' + podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' + + +######### Trial for podman ############# +# docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: VERSION,FROM_CACHE=true,BUILD_OPTS=[build options],EXAMPLE=true +# reg=$$(make _docker-get-reg) +# # Try to execute `make build` from the image directory +# if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) +# make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) +# exit +# elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# cd $(DOCKER_CUSTOM_DIR)/$(NAME) +# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) +# elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# cd $(DOCKER_DIR)/$(NAME) +# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) +# elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) +# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) +# fi +# # Dockerfile +# make NAME=$(NAME) \ +# docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ +# docker-image-set-version VERSION=$(VERSION) +# # Cache +# cache_from= +# if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then +# make docker-pull NAME=$(NAME) VERSION=latest +# cache_from="--cache-from $$reg/$(NAME):latest" +# fi +# # Build +# dir=$$(make _docker-get-dir) +# export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) +# export VERSION=$$(make docker-image-get-version) +# make -s file-replace-variables FILE=$$dir/Dockerfile.effective +# docker buildx build --rm \ +# --build-arg IMAGE=$$IMAGE \ +# --build-arg VERSION=$$VERSION \ +# --build-arg BUILD_ID=$(BUILD_ID) \ +# --build-arg BUILD_DATE=$(BUILD_DATE) \ +# --build-arg BUILD_REPO=$(BUILD_REPO) \ +# --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ +# --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ +# --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ +# --output type=docker \ +# --label name=$$IMAGE \ +# --label version=$$VERSION \ +# --label build-id=$(BUILD_ID) \ +# --label build-date=$(BUILD_DATE) \ +# --label build-repo=$(BUILD_REPO) \ +# --label build-branch=$(BUILD_BRANCH) \ +# --label build-commit-hash=$(BUILD_COMMIT_HASH) \ +# --label build-commit-date=$(BUILD_COMMIT_DATE) \ +# $(BUILD_OPTS) $$cache_from \ +# --file $$dir/Dockerfile.effective \ +# --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ +# $$dir +# # Tag +# docker tag \ +# $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ +# $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest +# docker rmi --force $$(docker images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: +# make docker-image-keep-latest-only NAME=$(NAME) +# docker image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' docker-test: ### Test image - mandatory: NAME; optional: ARGS,CMD,GOSS_OPTS,EXAMPLE=true dir=$$(make _docker-get-dir) From 884fed2ef490cc42afee2dbbc83198125e14a5f6 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Thu, 3 Apr 2025 12:01:12 +0100 Subject: [PATCH 02/11] Enabling use of podman --- Makefile | 52 ++++++++++++++++++++++++++++------ build/automation/lib/docker.mk | 13 +++++---- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index a69d381d8..becd8e178 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,8 @@ populate-tagging-variables: echo "export TF_VAR_distribution_list=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DISTRIBUTION_LIST_KEY)')" unit-test: - make -s docker-run-tester \ + # make -s docker-run-tester \ + make -s podman-run-tester \ CMD="python -m pytest application --junitxml=./testresults.xml --cov-report term-missing --cov-report xml:coverage.xml --cov=application -vv" \ ARGS=$(UNIT_TEST_ARGS) @@ -495,18 +496,20 @@ tag-commit-to-rollback-blue-green-environment: # Tags commit to rollback blue/gr commit-date-hash-tag: echo "$(BUILD_COMMIT_DATETIME)-$(BUILD_COMMIT_HASH)" -docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],LIB_VOLUME_MOUNT=true,VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] +######### Trial for podman ############# + +podman-run-tester: ### Run Python container with Podman make docker-config > /dev/null 2>&1 mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} - lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) - container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) - docker run --interactive $(_TTY) --rm \ + lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume=$(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume=$(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) + container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date -u +"%Y%m%d%H%M%S")-$$(make secret-random LENGTH=8)) + /opt/podman/bin/podman run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ - --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ - --env-file <(make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)") \ - --env-file <(make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)") \ - --env-file <(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE)) \ + --env-file /tmp/env_aws_tx.env \ + --env-file /tmp/env_db_api.env \ + --env-file /tmp/env_profile.env \ + --env-file /tmp/env_custom.env \ --env HOME=/tmp \ --env PIP_TARGET=/tmp/.packages \ --env PYTHONPATH=/tmp/.packages \ @@ -524,6 +527,37 @@ docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true, $(CMD) +######### Trial for podman ############# + +# docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],LIB_VOLUME_MOUNT=true,VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] +# make docker-config > /dev/null 2>&1 +# mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} +# lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) +# container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) +# docker run --interactive $(_TTY) --rm \ +# --name $$container \ +# --user $$(id -u):$$(id -g) \ +# --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ +# --env-file <(make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)") \ +# --env-file <(make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)") \ +# --env-file <(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE)) \ +# --env HOME=/tmp \ +# --env PIP_TARGET=/tmp/.packages \ +# --env PYTHONPATH=/tmp/.packages \ +# --env XDG_CACHE_HOME=/tmp/.cache \ +# --volume $(PROJECT_DIR):/project \ +# --volume $(HOME)/.aws:/tmp/.aws \ +# --volume $(HOME)/bin:/tmp/bin \ +# --volume $(HOME)/etc:/tmp/etc \ +# --volume $(HOME)/usr:/tmp/usr \ +# $$lib_volume_mount \ +# --network $(DOCKER_NETWORK) \ +# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ +# $(ARGS) \ +# $$(make _docker-get-reg)/tester:latest \ +# $(CMD) + + # ============================================================================== # Python Targets (including Ruff) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 8fb725cf4..a783672f5 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -5,7 +5,8 @@ DOCKER_LIB_DIR = $(LIB_DIR)/docker DOCKER_LIB_DIR_REL = $(shell echo $(DOCKER_LIB_DIR) | sed "s;$(PROJECT_DIR);;g") DOCKER_LIB_IMAGE_DIR = $(LIB_DIR)/docker/image DOCKER_LIB_IMAGE_DIR_REL = $(shell echo $(DOCKER_LIB_IMAGE_DIR) | sed "s;$(PROJECT_DIR);;g") -DOCKER_NETWORK = $(PROJECT_GROUP_SHORT)/$(PROJECT_NAME_SHORT)/$(BUILD_ID) +# DOCKER_NETWORK = $(PROJECT_GROUP_SHORT)/$(PROJECT_NAME_SHORT)/$(BUILD_ID) +DOCKER_NETWORK = $(PROJECT_GROUP_SHORT)_$(PROJECT_NAME_SHORT)_$(BUILD_ID) DOCKER_REGISTRY = $(AWS_ECR)/$(PROJECT_GROUP_SHORT)/$(PROJECT_NAME_SHORT) DOCKER_LIBRARY_REGISTRY = nhsd @@ -93,7 +94,7 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V # Cache cache_from= if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - make podman-pull NAME=$(NAME) VERSION=latest + make /opt/podman/bin/podman-pull NAME=$(NAME) VERSION=latest cache_from="--cache-from $$reg/$(NAME):latest" fi # Build @@ -101,7 +102,7 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) export VERSION=$$(make docker-image-get-version) make -s file-replace-variables FILE=$$dir/Dockerfile.effective - podman build --rm \ + /opt/podman/bin/podman build --rm \ --build-arg IMAGE=$$IMAGE \ --build-arg VERSION=$$VERSION \ --build-arg BUILD_ID=$(BUILD_ID) \ @@ -123,12 +124,12 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ $$dir # Tag - podman tag \ + /opt/podman/bin/podman tag \ $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest - podman rmi --force $$(podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: + /opt/podman/bin/podman rmi --force $$(/opt/podman/bin/podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: make docker-image-keep-latest-only NAME=$(NAME) - podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' + /opt/podman/bin/podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' ######### Trial for podman ############# From 1bda427e1ebb0a67af116f27e0d41e8cd6e3aea2 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Tue, 8 Apr 2025 16:25:02 +0100 Subject: [PATCH 03/11] Enabling use of podman --- Makefile | 82 +++++++------ build/automation/lib/docker.mk | 209 +++++++++++++++++---------------- 2 files changed, 154 insertions(+), 137 deletions(-) diff --git a/Makefile b/Makefile index becd8e178..c3cbf620a 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,16 @@ include $(abspath $(PROJECT_DIR)/build/automation/init.mk) # ============================================================================== # Development workflow targets +docker_cmd: + DOCKER_CMD=$(DOCKER_CMD) + +# docker_get_reg: +# echo $(_docker-get-reg) + +# build_ts: +# echo $(or $(BUILD_DATE), $(shell date -u +"%Y-%m-%dT%H:%M:%S%z")) +# echo $(shell date -j -f "%Y-%m-%dT%H:%M:%S%z" "$(BUILD_DATE)" -u +"%Y%m%d%H%M%S" 2>/dev/null) + setup: project-config # Set up project make tester-build @@ -51,8 +61,7 @@ populate-tagging-variables: echo "export TF_VAR_distribution_list=$$(echo $$DEPLOYMENT_SECRETS | jq -r '.$(DISTRIBUTION_LIST_KEY)')" unit-test: - # make -s docker-run-tester \ - make -s podman-run-tester \ + make -s docker-run-tester \ CMD="python -m pytest application --junitxml=./testresults.xml --cov-report term-missing --cov-report xml:coverage.xml --cov=application -vv" \ ARGS=$(UNIT_TEST_ARGS) @@ -336,8 +345,7 @@ batch-delete-ecr-images: # Mandatory - LIST_OF_DIGESTS: [list of "sha:digest" se tester-build: ### Build tester docker image cat $(APPLICATION_DIR)/*/requirements.txt $(APPLICATION_DIR)/requirements-dev.txt | sort --unique > $(DOCKER_DIR)/tester/assets/requirements.txt - # make -s docker-image NAME=tester - make -s podman-image NAME=tester + make -s docker-image NAME=tester tester-clean: rm -fv $(DOCKER_DIR)/tester/assets/*.txt @@ -498,43 +506,12 @@ commit-date-hash-tag: ######### Trial for podman ############# -podman-run-tester: ### Run Python container with Podman - make docker-config > /dev/null 2>&1 - mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} - lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume=$(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume=$(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) - container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date -u +"%Y%m%d%H%M%S")-$$(make secret-random LENGTH=8)) - /opt/podman/bin/podman run --interactive $(_TTY) --rm \ - --name $$container \ - --user $$(id -u):$$(id -g) \ - --env-file /tmp/env_aws_tx.env \ - --env-file /tmp/env_db_api.env \ - --env-file /tmp/env_profile.env \ - --env-file /tmp/env_custom.env \ - --env HOME=/tmp \ - --env PIP_TARGET=/tmp/.packages \ - --env PYTHONPATH=/tmp/.packages \ - --env XDG_CACHE_HOME=/tmp/.cache \ - --volume $(PROJECT_DIR):/project \ - --volume $(HOME)/.aws:/tmp/.aws \ - --volume $(HOME)/bin:/tmp/bin \ - --volume $(HOME)/etc:/tmp/etc \ - --volume $(HOME)/usr:/tmp/usr \ - $$lib_volume_mount \ - --network $(DOCKER_NETWORK) \ - --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ - $(ARGS) \ - $$(make _docker-get-reg)/tester:latest \ - $(CMD) - - -######### Trial for podman ############# - -# docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],LIB_VOLUME_MOUNT=true,VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] +# podman-run-tester: ### Run Python container with Podman # make docker-config > /dev/null 2>&1 # mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} # lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) # container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) -# docker run --interactive $(_TTY) --rm \ +# /opt/podman/bin/podman run --interactive $(_TTY) --rm \ # --name $$container \ # --user $$(id -u):$$(id -g) \ # --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ @@ -558,6 +535,37 @@ podman-run-tester: ### Run Python container with Podman # $(CMD) +######### Trial for podman ############# + +docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],LIB_VOLUME_MOUNT=true,VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] + make docker-config > /dev/null 2>&1 + mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} + lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) + container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ + --name $$container \ + --user $$(id -u):$$(id -g) \ + --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ + --env-file <(make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)") \ + --env-file <(make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)") \ + --env-file <(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE)) \ + --env HOME=/tmp \ + --env PIP_TARGET=/tmp/.packages \ + --env PYTHONPATH=/tmp/.packages \ + --env XDG_CACHE_HOME=/tmp/.cache \ + --volume $(PROJECT_DIR):/project \ + --volume $(HOME)/.aws:/tmp/.aws \ + --volume $(HOME)/bin:/tmp/bin \ + --volume $(HOME)/etc:/tmp/etc \ + --volume $(HOME)/usr:/tmp/usr \ + $$lib_volume_mount \ + --network $(DOCKER_NETWORK) \ + --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ + $(ARGS) \ + $$(make _docker-get-reg)/tester:latest \ + $(CMD) + + # ============================================================================== # Python Targets (including Ruff) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index a783672f5..5cc2b103f 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -44,6 +44,13 @@ DOCKER_LIBRARY_TOOLS_VERSION = $(shell cat $(DOCKER_LIB_IMAGE_DIR)/tools/VERSION COMPOSE_HTTP_TIMEOUT := $(or $(COMPOSE_HTTP_TIMEOUT), 6000) DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) +######### Trial for podman ############# + +# DOCKER_CMD=$(shell type -p docker >/dev/null 2>&1 && echo docker || echo podman) +DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) + +######### Trial for podman ############# + # ============================================================================== docker-create-from-template: ### Create Docker image from template - mandatory: NAME,TEMPLATE=[library template image name] @@ -66,98 +73,35 @@ docker-create-from-template: ### Create Docker image from template - mandatory: # ============================================================================== docker-config: ### Configure Docker networking - docker network create $(DOCKER_NETWORK) 2> /dev/null ||: + $(DOCKER_CMD) network create $(DOCKER_NETWORK) 2> /dev/null ||: ######### Trial for podman ############# -podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: VERSION, FROM_CACHE=true, BUILD_OPTS=[build options], EXAMPLE=true - reg=$$(make _docker-get-reg) - # Try to execute `make build` from the image directory - if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then - cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) - make build __PODMAN_BUILD=true PODMAN_REGISTRY=$(PODMAN_LIBRARY_REGISTRY) - exit - elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then - cd $(DOCKER_CUSTOM_DIR)/$(NAME) - make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) - elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then - cd $(DOCKER_DIR)/$(NAME) - make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) - elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then - cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) - make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) - fi - # Dockerfile (now Podman) - make NAME=$(NAME) \ - docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ - docker-image-set-version VERSION=$(VERSION) - # Cache - cache_from= - if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - make /opt/podman/bin/podman-pull NAME=$(NAME) VERSION=latest - cache_from="--cache-from $$reg/$(NAME):latest" - fi - # Build - dir=$$(make _docker-get-dir) - export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) - export VERSION=$$(make docker-image-get-version) - make -s file-replace-variables FILE=$$dir/Dockerfile.effective - /opt/podman/bin/podman build --rm \ - --build-arg IMAGE=$$IMAGE \ - --build-arg VERSION=$$VERSION \ - --build-arg BUILD_ID=$(BUILD_ID) \ - --build-arg BUILD_DATE=$(BUILD_DATE) \ - --build-arg BUILD_REPO=$(BUILD_REPO) \ - --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ - --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ - --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ - --label name=$$IMAGE \ - --label version=$$VERSION \ - --label build-id=$(BUILD_ID) \ - --label build-date=$(BUILD_DATE) \ - --label build-repo=$(BUILD_REPO) \ - --label build-branch=$(BUILD_BRANCH) \ - --label build-commit-hash=$(BUILD_COMMIT_HASH) \ - --label build-commit-date=$(BUILD_COMMIT_DATE) \ - $(BUILD_OPTS) $$cache_from \ - --file $$dir/Dockerfile.effective \ - --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ - $$dir - # Tag - /opt/podman/bin/podman tag \ - $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ - $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest - /opt/podman/bin/podman rmi --force $$(/opt/podman/bin/podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: - make docker-image-keep-latest-only NAME=$(NAME) - /opt/podman/bin/podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' - - -######### Trial for podman ############# -# docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: VERSION,FROM_CACHE=true,BUILD_OPTS=[build options],EXAMPLE=true +# podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: VERSION, FROM_CACHE=true, BUILD_OPTS=[build options], EXAMPLE=true # reg=$$(make _docker-get-reg) # # Try to execute `make build` from the image directory -# if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then # cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) -# make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) +# make build __PODMAN_BUILD=true PODMAN_REGISTRY=$(PODMAN_LIBRARY_REGISTRY) # exit -# elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then # cd $(DOCKER_CUSTOM_DIR)/$(NAME) -# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) -# elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) +# elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then # cd $(DOCKER_DIR)/$(NAME) -# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) -# elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then +# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) +# elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then # cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) -# make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) +# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) # fi -# # Dockerfile +# # Dockerfile (now Podman) # make NAME=$(NAME) \ # docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ # docker-image-set-version VERSION=$(VERSION) # # Cache # cache_from= # if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then -# make docker-pull NAME=$(NAME) VERSION=latest +# make /opt/podman/bin/podman-pull NAME=$(NAME) VERSION=latest # cache_from="--cache-from $$reg/$(NAME):latest" # fi # # Build @@ -165,7 +109,7 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V # export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) # export VERSION=$$(make docker-image-get-version) # make -s file-replace-variables FILE=$$dir/Dockerfile.effective -# docker buildx build --rm \ +# /opt/podman/bin/podman build --rm \ # --build-arg IMAGE=$$IMAGE \ # --build-arg VERSION=$$VERSION \ # --build-arg BUILD_ID=$(BUILD_ID) \ @@ -174,7 +118,6 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V # --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ # --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ # --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ -# --output type=docker \ # --label name=$$IMAGE \ # --label version=$$VERSION \ # --label build-id=$(BUILD_ID) \ @@ -188,12 +131,75 @@ podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: V # --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ # $$dir # # Tag -# docker tag \ +# /opt/podman/bin/podman tag \ # $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ # $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest -# docker rmi --force $$(docker images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: +# /opt/podman/bin/podman rmi --force $$(/opt/podman/bin/podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: # make docker-image-keep-latest-only NAME=$(NAME) -# docker image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' +# /opt/podman/bin/podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' + + +######### Trial for podman ############# +docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: VERSION,FROM_CACHE=true,BUILD_OPTS=[build options],EXAMPLE=true + reg=$$(make _docker-get-reg) + # Try to execute `make build` from the image directory + if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) + make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) + exit + elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + cd $(DOCKER_CUSTOM_DIR)/$(NAME) + make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) + elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + cd $(DOCKER_DIR)/$(NAME) + make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) + elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) + make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) + fi + # Dockerfile + make NAME=$(NAME) \ + docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ + docker-image-set-version VERSION=$(VERSION) + # Cache + cache_from= + if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then + make docker-pull NAME=$(NAME) VERSION=latest + cache_from="--cache-from $$reg/$(NAME):latest" + fi + # Build + dir=$$(make _docker-get-dir) + export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) + export VERSION=$$(make docker-image-get-version) + make -s file-replace-variables FILE=$$dir/Dockerfile.effective + $(DOCKER_CMD) buildx build --rm \ + --build-arg IMAGE=$$IMAGE \ + --build-arg VERSION=$$VERSION \ + --build-arg BUILD_ID=$(BUILD_ID) \ + --build-arg BUILD_DATE=$(BUILD_DATE) \ + --build-arg BUILD_REPO=$(BUILD_REPO) \ + --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ + --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ + --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ + --label name=$$IMAGE \ + --label version=$$VERSION \ + --label build-id=$(BUILD_ID) \ + --label build-date=$(BUILD_DATE) \ + --label build-repo=$(BUILD_REPO) \ + --label build-branch=$(BUILD_BRANCH) \ + --label build-commit-hash=$(BUILD_COMMIT_HASH) \ + --label build-commit-date=$(BUILD_COMMIT_DATE) \ + $(BUILD_OPTS) $$cache_from \ + --file $$dir/Dockerfile.effective \ + --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ + $$dir + # Tag + $(DOCKER_CMD) tag \ + $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ + $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest + $(DOCKER_CMD) rmi --force $$($(DOCKER_CMD) images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: + make docker-image-keep-latest-only NAME=$(NAME) + $(DOCKER_CMD) image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' docker-test: ### Test image - mandatory: NAME; optional: ARGS,CMD,GOSS_OPTS,EXAMPLE=true dir=$$(make _docker-get-dir) @@ -209,9 +215,9 @@ docker-test: ### Test image - mandatory: NAME; optional: ARGS,CMD,GOSS_OPTS,EXAM docker-login: ### Log into the Docker registry - optional: DOCKER_USERNAME,DOCKER_PASSWORD if [ -n "$(DOCKER_USERNAME)" ] && [ -n "$$(make _docker-get-login-password)" ]; then - make _docker-get-login-password | docker login --username "$(DOCKER_USERNAME)" --password-stdin + make _docker-get-login-password | $(DOCKER_CMD) login --username "$(DOCKER_USERNAME)" --password-stdin else - make aws-ecr-get-login-password | docker login --username AWS --password-stdin $(AWS_ECR) + make aws-ecr-get-login-password | $(DOCKER_CMD) login --username AWS --password-stdin $(AWS_ECR) fi docker-create-repository: ### Create Docker repository to store an image - mandatory: NAME; optional: POLICY_FILE=[policy file] @@ -221,11 +227,14 @@ docker-push: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG make docker-login reg=$$(make _docker-get-reg) if [ -n "$(or $(VERSION), $(TAG))" ]; then - docker push $$reg/$(NAME):$(or $(VERSION), $(TAG)) + echo "1 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" + $(DOCKER_CMD) push $$reg/$(NAME):$(or $(VERSION), $(TAG)) else - docker push $$reg/$(NAME):$$(make docker-image-get-version) + echo "2 -----------------------------------------------------------------$(NAME) $$(make docker-image-get-version)" + $(DOCKER_CMD) push $$reg/$(NAME):$$(make docker-image-get-version) fi - docker push $$reg/$(NAME):latest 2> /dev/null ||: + echo "3 -----------------------------------------------------------------$(NAME)" + $(DOCKER_CMD) push $$reg/$(NAME):latest 2> /dev/null ||: docker-push-for-production: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG make docker-login @@ -240,23 +249,23 @@ docker-pull: ### Pull Docker image - mandatory: NAME,DIGEST|VERSION|TAG [ $$(make _docker-is-lib-image) == false ] && make docker-login reg=$$(make _docker-get-reg) if [ -n "$(DIGEST)" ]; then - docker pull $$reg/$(NAME)@$(DIGEST) ||: + $(DOCKER_CMD) pull $$reg/$(NAME)@$(DIGEST) ||: else - docker pull $$reg/$(NAME):$(or $(VERSION), $(TAG)) ||: + $(DOCKER_CMD) pull $$reg/$(NAME):$(or $(VERSION), $(TAG)) ||: fi docker-tag: ### Tag latest or provide arguments - mandatory: NAME,VERSION|TAG|[SOURCE,TARGET]|[DIGEST,VERSION|TAG] reg=$$(make _docker-get-reg) if [ -n "$(DIGEST)" ] && [ -n "$(TAG)" ]; then - docker tag \ + $(DOCKER_CMD) tag \ $$reg/$(NAME)@$(DIGEST) \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) elif [ -n "$(SOURCE)" ] && [ -n "$(TARGET)" ]; then - docker tag \ + $(DOCKER_CMD) tag \ $$reg/$(NAME):$(SOURCE) \ $$reg/$(NAME):$(TARGET) elif [ -n "$(or $(VERSION), $(TAG))" ]; then - docker tag \ + $(DOCKER_CMD) tag \ $$reg/$(NAME):latest \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) fi @@ -346,25 +355,25 @@ docker-image-set-version: ### Set effective Docker image version - mandatory: NA docker-image-pull-or-build: ### Pull or build image - mandatory: NAME; optional VERSION|TAG=[defaults to 'latest'],LATEST=true version=$(or $(or $(VERSION), $(TAG)), latest) image=$(DOCKER_LIBRARY_REGISTRY)/$(NAME):$$version - if [ -z "$$(docker images --filter=reference="$$image" --quiet)" ]; then + if [ -z "$$($(DOCKER_CMD) images --filter=reference="$$image" --quiet)" ]; then make docker-pull NAME=$(NAME) VERSION=$$version ||: fi - if [ -z "$$(docker images --filter=reference="$$image" --quiet)" ]; then + if [ -z "$$($(DOCKER_CMD) images --filter=reference="$$image" --quiet)" ]; then make docker-build NAME=$(NAME) || ( \ echo "ERROR: No image $$image found"; \ exit 1 \ ) fi - if [ -n "$$(docker images --filter=reference="$$image" --quiet)" ]; then + if [ -n "$$($(DOCKER_CMD) images --filter=reference="$$image" --quiet)" ]; then [[ "$(LATEST)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]] && \ make docker-tag NAME=$(NAME) SOURCE=$$version TARGET=latest ||: fi docker-image-keep-latest-only: ### Remove other images than latest - mandatory: NAME reg=$$(make _docker-get-reg) - docker rmi --force $$( \ - docker images --filter=reference="$$reg/$(NAME):*" --quiet | \ - grep -v $$(docker images --filter=reference="$$reg/$(NAME):latest" --quiet) \ + $(DOCKER_CMD) rmi --force $$( \ + $(DOCKER_CMD) images --filter=reference="$$reg/$(NAME):*" --quiet | \ + grep -v $$($(DOCKER_CMD) images --filter=reference="$$reg/$(NAME):latest" --quiet) \ ) 2> /dev/null ||: docker-image-start: ### Start container - mandatory: NAME; optional: CMD,DIR,ARGS=[Docker args],VARS_FILE=[Makefile vars file],EXAMPLE=true @@ -430,7 +439,7 @@ docker-run: ### Run specified image - mandatory: IMAGE; optional: CMD,SH=true,DI make docker-config > /dev/null 2>&1 container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo $$(echo '$(IMAGE)' | md5sum | cut -c1-7)-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) if [[ ! "$(SH)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ @@ -491,7 +500,7 @@ docker-run-editorconfig: ### Run editorconfig container - optional: DIR=[working make docker-config > /dev/null 2>&1 image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo mstruebing/editorconfig-checker:$(DOCKER_EDITORCONFIG_CHECKER_VERSION)) container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo editorconfig-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --volume $$([ -n "$(DIR)" ] && echo $(abspath $(DIR)) || echo $(PWD)):/check \ @@ -780,7 +789,7 @@ docker-run-tools: ### Run tools (Python) container - mandatory: CMD; optional: S container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tools-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) make docker-image-pull-or-build NAME=tools VERSION=$(DOCKER_LIBRARY_TOOLS_VERSION) >&2 if [[ ! "$(SH)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ @@ -803,7 +812,7 @@ docker-run-tools: ### Run tools (Python) container - mandatory: CMD; optional: S $$image \ $(CMD) else - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ From 95a867ef842c6aedac3b4e18127095025558739a Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Wed, 9 Apr 2025 15:00:46 +0100 Subject: [PATCH 04/11] Enabling use podman in build-and-push target --- Makefile | 38 ++-------------- build/automation/lib/docker.mk | 79 +++++----------------------------- 2 files changed, 14 insertions(+), 103 deletions(-) diff --git a/Makefile b/Makefile index c3cbf620a..ab35dc36e 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,12 @@ include $(abspath $(PROJECT_DIR)/build/automation/init.mk) # ============================================================================== # Development workflow targets -docker_cmd: +docker-cmd: DOCKER_CMD=$(DOCKER_CMD) +docker-debugger: + DOCKER_LIB_IMAGE_DIR=$(DOCKER_LIB_IMAGE_DIR) + # docker_get_reg: # echo $(_docker-get-reg) @@ -504,39 +507,6 @@ tag-commit-to-rollback-blue-green-environment: # Tags commit to rollback blue/gr commit-date-hash-tag: echo "$(BUILD_COMMIT_DATETIME)-$(BUILD_COMMIT_HASH)" -######### Trial for podman ############# - -# podman-run-tester: ### Run Python container with Podman -# make docker-config > /dev/null 2>&1 -# mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} -# lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" == true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) -# container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tester-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) -# /opt/podman/bin/podman run --interactive $(_TTY) --rm \ -# --name $$container \ -# --user $$(id -u):$$(id -g) \ -# --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ -# --env-file <(make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)") \ -# --env-file <(make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)") \ -# --env-file <(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE)) \ -# --env HOME=/tmp \ -# --env PIP_TARGET=/tmp/.packages \ -# --env PYTHONPATH=/tmp/.packages \ -# --env XDG_CACHE_HOME=/tmp/.cache \ -# --volume $(PROJECT_DIR):/project \ -# --volume $(HOME)/.aws:/tmp/.aws \ -# --volume $(HOME)/bin:/tmp/bin \ -# --volume $(HOME)/etc:/tmp/etc \ -# --volume $(HOME)/usr:/tmp/usr \ -# $$lib_volume_mount \ -# --network $(DOCKER_NETWORK) \ -# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ -# $(ARGS) \ -# $$(make _docker-get-reg)/tester:latest \ -# $(CMD) - - -######### Trial for podman ############# - docker-run-tester: ### Run python container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],LIB_VOLUME_MOUNT=true,VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] make docker-config > /dev/null 2>&1 mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 5cc2b103f..53b8d63d0 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -44,13 +44,9 @@ DOCKER_LIBRARY_TOOLS_VERSION = $(shell cat $(DOCKER_LIB_IMAGE_DIR)/tools/VERSION COMPOSE_HTTP_TIMEOUT := $(or $(COMPOSE_HTTP_TIMEOUT), 6000) DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) -######### Trial for podman ############# - # DOCKER_CMD=$(shell type -p docker >/dev/null 2>&1 && echo docker || echo podman) DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) -######### Trial for podman ############# - # ============================================================================== docker-create-from-template: ### Create Docker image from template - mandatory: NAME,TEMPLATE=[library template image name] @@ -75,75 +71,11 @@ docker-create-from-template: ### Create Docker image from template - mandatory: docker-config: ### Configure Docker networking $(DOCKER_CMD) network create $(DOCKER_NETWORK) 2> /dev/null ||: -######### Trial for podman ############# - -# podman-build podman-image: ### Build Podman image - mandatory: NAME; optional: VERSION, FROM_CACHE=true, BUILD_OPTS=[build options], EXAMPLE=true -# reg=$$(make _docker-get-reg) -# # Try to execute `make build` from the image directory -# if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then -# cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) -# make build __PODMAN_BUILD=true PODMAN_REGISTRY=$(PODMAN_LIBRARY_REGISTRY) -# exit -# elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then -# cd $(DOCKER_CUSTOM_DIR)/$(NAME) -# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) -# elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then -# cd $(DOCKER_DIR)/$(NAME) -# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) -# elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__PODMAN_BUILD)" ]; then -# cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) -# make build __PODMAN_BUILD=true && exit || cd $(PROJECT_DIR) -# fi -# # Dockerfile (now Podman) -# make NAME=$(NAME) \ -# docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ -# docker-image-set-version VERSION=$(VERSION) -# # Cache -# cache_from= -# if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then -# make /opt/podman/bin/podman-pull NAME=$(NAME) VERSION=latest -# cache_from="--cache-from $$reg/$(NAME):latest" -# fi -# # Build -# dir=$$(make _docker-get-dir) -# export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) -# export VERSION=$$(make docker-image-get-version) -# make -s file-replace-variables FILE=$$dir/Dockerfile.effective -# /opt/podman/bin/podman build --rm \ -# --build-arg IMAGE=$$IMAGE \ -# --build-arg VERSION=$$VERSION \ -# --build-arg BUILD_ID=$(BUILD_ID) \ -# --build-arg BUILD_DATE=$(BUILD_DATE) \ -# --build-arg BUILD_REPO=$(BUILD_REPO) \ -# --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ -# --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ -# --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ -# --label name=$$IMAGE \ -# --label version=$$VERSION \ -# --label build-id=$(BUILD_ID) \ -# --label build-date=$(BUILD_DATE) \ -# --label build-repo=$(BUILD_REPO) \ -# --label build-branch=$(BUILD_BRANCH) \ -# --label build-commit-hash=$(BUILD_COMMIT_HASH) \ -# --label build-commit-date=$(BUILD_COMMIT_DATE) \ -# $(BUILD_OPTS) $$cache_from \ -# --file $$dir/Dockerfile.effective \ -# --tag $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ -# $$dir -# # Tag -# /opt/podman/bin/podman tag \ -# $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):$$(make docker-image-get-version) \ -# $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest -# /opt/podman/bin/podman rmi --force $$(/opt/podman/bin/podman images | grep "" | awk '{ print $$3 }') 2> /dev/null ||: -# make docker-image-keep-latest-only NAME=$(NAME) -# /opt/podman/bin/podman image inspect $$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example):latest --format='{{.Size}}' - - -######### Trial for podman ############# docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: VERSION,FROM_CACHE=true,BUILD_OPTS=[build options],EXAMPLE=true reg=$$(make _docker-get-reg) # Try to execute `make build` from the image directory if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + echo "0_____________________________$(DOCKER_LIB_IMAGE_DIR), $(NAME), $(__DOCKER_BUILD), $(DOCKER_LIBRARY_REGISTRY)" cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) exit @@ -226,6 +158,7 @@ docker-create-repository: ### Create Docker repository to store an image - manda docker-push: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG make docker-login reg=$$(make _docker-get-reg) + echo "0 -----------------------------------------------------------------$(IMAGE_NAME) or $(IMAGE_VERSION) or $(IMAGE_TAG)" if [ -n "$(or $(VERSION), $(TAG))" ]; then echo "1 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" $(DOCKER_CMD) push $$reg/$(NAME):$(or $(VERSION), $(TAG)) @@ -257,14 +190,17 @@ docker-pull: ### Pull Docker image - mandatory: NAME,DIGEST|VERSION|TAG docker-tag: ### Tag latest or provide arguments - mandatory: NAME,VERSION|TAG|[SOURCE,TARGET]|[DIGEST,VERSION|TAG] reg=$$(make _docker-get-reg) if [ -n "$(DIGEST)" ] && [ -n "$(TAG)" ]; then + echo "0_____________________________$(DIGEST), $(NAME), $(VERSION), $(TAG)" $(DOCKER_CMD) tag \ $$reg/$(NAME)@$(DIGEST) \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) elif [ -n "$(SOURCE)" ] && [ -n "$(TARGET)" ]; then + echo "1_____________________________$(SOURCE), $(TARGET) " $(DOCKER_CMD) tag \ $$reg/$(NAME):$(SOURCE) \ $$reg/$(NAME):$(TARGET) elif [ -n "$(or $(VERSION), $(TAG))" ]; then + echo "2_____________________________$(NAME), $(VERSION), $(TAG)" $(DOCKER_CMD) tag \ $$reg/$(NAME):latest \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) @@ -876,13 +812,18 @@ docker-compose-exec: ### Run Docker Compose exec command - mandatory: CMD; optio # ============================================================================== _docker-get-dir: + echo "---------------------- NAME=$(NAME)" if [ -n "$(DOCKER_CUSTOM_DIR)" ] && [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ]; then + echo "0----------------------" echo $(DOCKER_CUSTOM_DIR)/$(NAME) elif [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ]; then + echo "1----------------------" echo $(DOCKER_LIB_IMAGE_DIR)/$(NAME) elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ ! -z $(GENERIC_IMAGE_NAME) ] ; then + echo "2----------------------" echo $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) else + echo "3----------------------" echo $(DOCKER_DIR)/$(NAME) fi From 6fae70ce6d316e68459f1588e7f03374aee0dd4a Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Wed, 9 Apr 2025 17:39:43 +0100 Subject: [PATCH 05/11] Enabling use of docker for podman instead of docker --- Makefile | 1 + build/automation/lib/docker.mk | 15 +++++++-------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index ab35dc36e..05a415961 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ build-lambda: ### Build lambda docker image - mandatory: NAME build-and-push: # Build lambda docker images and pushes them to ECR for IMAGE_NAME in $$(echo $(PROJECT_LAMBDAS_LIST) | tr "," "\n"); do make -s build-lambda GENERIC_IMAGE_NAME=lambda NAME=$$IMAGE_NAME + echo $$IMAGE_NAME+"1" make -s docker-push NAME=$$IMAGE_NAME done diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 53b8d63d0..a56bf38dc 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -44,7 +44,6 @@ DOCKER_LIBRARY_TOOLS_VERSION = $(shell cat $(DOCKER_LIB_IMAGE_DIR)/tools/VERSION COMPOSE_HTTP_TIMEOUT := $(or $(COMPOSE_HTTP_TIMEOUT), 6000) DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) -# DOCKER_CMD=$(shell type -p docker >/dev/null 2>&1 && echo docker || echo podman) DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) # ============================================================================== @@ -75,17 +74,20 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V reg=$$(make _docker-get-reg) # Try to execute `make build` from the image directory if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then - echo "0_____________________________$(DOCKER_LIB_IMAGE_DIR), $(NAME), $(__DOCKER_BUILD), $(DOCKER_LIBRARY_REGISTRY)" + echo "00------------------------------------------------------------------" cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) exit elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + echo "01------------------------------------------------------------------" cd $(DOCKER_CUSTOM_DIR)/$(NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + echo "02------------------------------------------------------------------" cd $(DOCKER_DIR)/$(NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then + echo "03------------------------------------------------------------------" cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) fi @@ -93,6 +95,7 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V make NAME=$(NAME) \ docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ docker-image-set-version VERSION=$(VERSION) + echo "VERSION----------------------------------------------$(VERSION)" # Cache cache_from= if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then @@ -103,6 +106,7 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V dir=$$(make _docker-get-dir) export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) export VERSION=$$(make docker-image-get-version) + echo "$$(make docker-image-get-version)--------------------------" make -s file-replace-variables FILE=$$dir/Dockerfile.effective $(DOCKER_CMD) buildx build --rm \ --build-arg IMAGE=$$IMAGE \ @@ -158,7 +162,7 @@ docker-create-repository: ### Create Docker repository to store an image - manda docker-push: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG make docker-login reg=$$(make _docker-get-reg) - echo "0 -----------------------------------------------------------------$(IMAGE_NAME) or $(IMAGE_VERSION) or $(IMAGE_TAG)" + echo "0 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" if [ -n "$(or $(VERSION), $(TAG))" ]; then echo "1 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" $(DOCKER_CMD) push $$reg/$(NAME):$(or $(VERSION), $(TAG)) @@ -812,18 +816,13 @@ docker-compose-exec: ### Run Docker Compose exec command - mandatory: CMD; optio # ============================================================================== _docker-get-dir: - echo "---------------------- NAME=$(NAME)" if [ -n "$(DOCKER_CUSTOM_DIR)" ] && [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ]; then - echo "0----------------------" echo $(DOCKER_CUSTOM_DIR)/$(NAME) elif [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ]; then - echo "1----------------------" echo $(DOCKER_LIB_IMAGE_DIR)/$(NAME) elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ ! -z $(GENERIC_IMAGE_NAME) ] ; then - echo "2----------------------" echo $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) else - echo "3----------------------" echo $(DOCKER_DIR)/$(NAME) fi From 167a061dbdd017c3c2ca02b7553425c4e4255a30 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Thu, 10 Apr 2025 16:59:07 +0100 Subject: [PATCH 06/11] Running make targets using podman --- Makefile | 1 - build/automation/init.mk | 2 +- build/automation/lib/docker.mk | 7 ++++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 05a415961..ab35dc36e 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,6 @@ build-lambda: ### Build lambda docker image - mandatory: NAME build-and-push: # Build lambda docker images and pushes them to ECR for IMAGE_NAME in $$(echo $(PROJECT_LAMBDAS_LIST) | tr "," "\n"); do make -s build-lambda GENERIC_IMAGE_NAME=lambda NAME=$$IMAGE_NAME - echo $$IMAGE_NAME+"1" make -s docker-push NAME=$$IMAGE_NAME done diff --git a/build/automation/init.mk b/build/automation/init.mk index b40d4e8e9..e129be6f1 100644 --- a/build/automation/init.mk +++ b/build/automation/init.mk @@ -617,7 +617,7 @@ GIT_COMMIT_MESSAGE_MAX_LENGTH := 72 GIT_PULL_REQUEST_TITLE_MAX_LENGTH := $(shell echo $$(( $(GIT_COMMIT_MESSAGE_MAX_LENGTH) + 12 ))) BUILD_DATE := $(or $(BUILD_DATE), $(shell date -u +"%Y-%m-%dT%H:%M:%S%z")) -BUILD_TIMESTAMP := $(shell date --date=$(BUILD_DATE) -u +"%Y%m%d%H%M%S" 2> /dev/null) +BUILD_TIMESTAMP := $(shell gdate --date=$(BUILD_DATE) -u +"%Y%m%d%H%M%S" 2> /dev/null) BUILD_ID := $(or $(or $(or $(or $(shell ([ -z "$(BUILD_ID)" ] && [ -n "$(JENKINS_URL)" ]) && echo $(BUILD_TIMESTAMP) ||:), $(BUILD_ID)), $(CIRCLE_BUILD_NUM)), $(CODEBUILD_BUILD_NUMBER)), 0) BUILD_REPO := $(or $(shell git config --get remote.origin.url 2> /dev/null ||:), unknown) BUILD_BRANCH := $(if $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null | grep -E ^HEAD$ ||:),$(or $(shell git name-rev --name-only HEAD 2> /dev/null | sed "s;remotes/origin/;;g" ||:), unknown),$(or $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null | sed "s;remotes/origin/;;g" ||:), unknown)) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index a56bf38dc..ef582dbdf 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -270,14 +270,18 @@ docker-image-get-version: ### Get effective Docker image version - mandatory: NA cat $$dir/.version 2> /dev/null || cat $$dir/VERSION 2> /dev/null || echo unknown docker-image-set-version: ### Set effective Docker image version - mandatory: NAME; optional: VERSION + echo "-----------------printing variables to get the .version file location $(DOCKER_LIB_IMAGE_DIR), $(NAME), $(DOCKER_CUSTOM_DIR), $(VERSION)" if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(DOCKER_CUSTOM_DIR)" ]; then + echo "000---------------------------Inside set image version if $(DOCKER_LIB_IMAGE_DIR)/$(NAME)/.version" rm -f $(DOCKER_LIB_IMAGE_DIR)/$(NAME)/.version exit fi dir=$$(make _docker-get-dir) if [ -n "$(VERSION)" ]; then + echo "001---------------------------Inside set image version if $(VERSION), $$(make _docker-get-dir)" echo $(VERSION) > $$dir/.version else + echo "002---------------------------Inside set image version else $$(cat $$dir/VERSION)" echo $$(cat $$dir/VERSION) | \ sed "s/YYYY/$$(date --date=$(BUILD_DATE) -u +"%Y")/g" | \ sed "s/mm/$$(date --date=$(BUILD_DATE) -u +"%m")/g" | \ @@ -288,6 +292,7 @@ docker-image-set-version: ### Set effective Docker image version - mandatory: NA sed "s/SS/$$(date --date=$(BUILD_DATE) -u +"%S")/g" | \ sed "s/hash/$$(git rev-parse --short HEAD)/g" \ > $$dir/.version + echo "-------------------------------------dir/.version = $$(cat $$dir/.version)" fi # ============================================================================== @@ -609,7 +614,7 @@ docker-run-terraform: ### Run terraform container - mandatory: CMD; optional: DI make docker-config > /dev/null 2>&1 image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo hashicorp/terraform:$(DOCKER_TERRAFORM_VERSION)) container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo terraform-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ From db064e8513a9a4bb2c89269ded93d4e070dfe94f Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Tue, 15 Apr 2025 17:23:02 +0100 Subject: [PATCH 07/11] Enabling podman for make targets --- Makefile | 13 ------------- build/automation/init.mk | 2 +- build/automation/lib/docker.mk | 22 ++-------------------- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index ab35dc36e..b75be05c9 100644 --- a/Makefile +++ b/Makefile @@ -4,19 +4,6 @@ include $(abspath $(PROJECT_DIR)/build/automation/init.mk) # ============================================================================== # Development workflow targets -docker-cmd: - DOCKER_CMD=$(DOCKER_CMD) - -docker-debugger: - DOCKER_LIB_IMAGE_DIR=$(DOCKER_LIB_IMAGE_DIR) - -# docker_get_reg: -# echo $(_docker-get-reg) - -# build_ts: -# echo $(or $(BUILD_DATE), $(shell date -u +"%Y-%m-%dT%H:%M:%S%z")) -# echo $(shell date -j -f "%Y-%m-%dT%H:%M:%S%z" "$(BUILD_DATE)" -u +"%Y%m%d%H%M%S" 2>/dev/null) - setup: project-config # Set up project make tester-build diff --git a/build/automation/init.mk b/build/automation/init.mk index e129be6f1..b40d4e8e9 100644 --- a/build/automation/init.mk +++ b/build/automation/init.mk @@ -617,7 +617,7 @@ GIT_COMMIT_MESSAGE_MAX_LENGTH := 72 GIT_PULL_REQUEST_TITLE_MAX_LENGTH := $(shell echo $$(( $(GIT_COMMIT_MESSAGE_MAX_LENGTH) + 12 ))) BUILD_DATE := $(or $(BUILD_DATE), $(shell date -u +"%Y-%m-%dT%H:%M:%S%z")) -BUILD_TIMESTAMP := $(shell gdate --date=$(BUILD_DATE) -u +"%Y%m%d%H%M%S" 2> /dev/null) +BUILD_TIMESTAMP := $(shell date --date=$(BUILD_DATE) -u +"%Y%m%d%H%M%S" 2> /dev/null) BUILD_ID := $(or $(or $(or $(or $(shell ([ -z "$(BUILD_ID)" ] && [ -n "$(JENKINS_URL)" ]) && echo $(BUILD_TIMESTAMP) ||:), $(BUILD_ID)), $(CIRCLE_BUILD_NUM)), $(CODEBUILD_BUILD_NUMBER)), 0) BUILD_REPO := $(or $(shell git config --get remote.origin.url 2> /dev/null ||:), unknown) BUILD_BRANCH := $(if $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null | grep -E ^HEAD$ ||:),$(or $(shell git name-rev --name-only HEAD 2> /dev/null | sed "s;remotes/origin/;;g" ||:), unknown),$(or $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null | sed "s;remotes/origin/;;g" ||:), unknown)) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index ef582dbdf..6b258a721 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -74,20 +74,16 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V reg=$$(make _docker-get-reg) # Try to execute `make build` from the image directory if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then - echo "00------------------------------------------------------------------" cd $(DOCKER_LIB_IMAGE_DIR)/$(NAME) make build __DOCKER_BUILD=true DOCKER_REGISTRY=$(DOCKER_LIBRARY_REGISTRY) exit elif [ -d $(DOCKER_CUSTOM_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then - echo "01------------------------------------------------------------------" cd $(DOCKER_CUSTOM_DIR)/$(NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) elif [ -d $(DOCKER_DIR)/$(NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then - echo "02------------------------------------------------------------------" cd $(DOCKER_DIR)/$(NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) elif [ -d $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) ] && [ -z "$(__DOCKER_BUILD)" ]; then - echo "03------------------------------------------------------------------" cd $(DOCKER_DIR)/$(GENERIC_IMAGE_NAME) make build __DOCKER_BUILD=true && exit || cd $(PROJECT_DIR) fi @@ -95,7 +91,6 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V make NAME=$(NAME) \ docker-create-dockerfile FILE=Dockerfile$(shell [ -n "$(EXAMPLE)" ] && echo .example) \ docker-image-set-version VERSION=$(VERSION) - echo "VERSION----------------------------------------------$(VERSION)" # Cache cache_from= if [[ "$(FROM_CACHE)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then @@ -106,7 +101,6 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V dir=$$(make _docker-get-dir) export IMAGE=$$reg/$(NAME)$(shell [ -n "$(EXAMPLE)" ] && echo -example) export VERSION=$$(make docker-image-get-version) - echo "$$(make docker-image-get-version)--------------------------" make -s file-replace-variables FILE=$$dir/Dockerfile.effective $(DOCKER_CMD) buildx build --rm \ --build-arg IMAGE=$$IMAGE \ @@ -162,15 +156,11 @@ docker-create-repository: ### Create Docker repository to store an image - manda docker-push: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG make docker-login reg=$$(make _docker-get-reg) - echo "0 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" if [ -n "$(or $(VERSION), $(TAG))" ]; then - echo "1 -----------------------------------------------------------------$(NAME) or $(VERSION) or $(TAG)" $(DOCKER_CMD) push $$reg/$(NAME):$(or $(VERSION), $(TAG)) else - echo "2 -----------------------------------------------------------------$(NAME) $$(make docker-image-get-version)" $(DOCKER_CMD) push $$reg/$(NAME):$$(make docker-image-get-version) fi - echo "3 -----------------------------------------------------------------$(NAME)" $(DOCKER_CMD) push $$reg/$(NAME):latest 2> /dev/null ||: docker-push-for-production: ### Push Docker image - mandatory: NAME; optional: VERSION|TAG @@ -194,17 +184,14 @@ docker-pull: ### Pull Docker image - mandatory: NAME,DIGEST|VERSION|TAG docker-tag: ### Tag latest or provide arguments - mandatory: NAME,VERSION|TAG|[SOURCE,TARGET]|[DIGEST,VERSION|TAG] reg=$$(make _docker-get-reg) if [ -n "$(DIGEST)" ] && [ -n "$(TAG)" ]; then - echo "0_____________________________$(DIGEST), $(NAME), $(VERSION), $(TAG)" $(DOCKER_CMD) tag \ $$reg/$(NAME)@$(DIGEST) \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) elif [ -n "$(SOURCE)" ] && [ -n "$(TARGET)" ]; then - echo "1_____________________________$(SOURCE), $(TARGET) " $(DOCKER_CMD) tag \ $$reg/$(NAME):$(SOURCE) \ $$reg/$(NAME):$(TARGET) elif [ -n "$(or $(VERSION), $(TAG))" ]; then - echo "2_____________________________$(NAME), $(VERSION), $(TAG)" $(DOCKER_CMD) tag \ $$reg/$(NAME):latest \ $$reg/$(NAME):$(or $(VERSION), $(TAG)) @@ -270,18 +257,14 @@ docker-image-get-version: ### Get effective Docker image version - mandatory: NA cat $$dir/.version 2> /dev/null || cat $$dir/VERSION 2> /dev/null || echo unknown docker-image-set-version: ### Set effective Docker image version - mandatory: NAME; optional: VERSION - echo "-----------------printing variables to get the .version file location $(DOCKER_LIB_IMAGE_DIR), $(NAME), $(DOCKER_CUSTOM_DIR), $(VERSION)" if [ -d $(DOCKER_LIB_IMAGE_DIR)/$(NAME) ] && [ -z "$(DOCKER_CUSTOM_DIR)" ]; then - echo "000---------------------------Inside set image version if $(DOCKER_LIB_IMAGE_DIR)/$(NAME)/.version" rm -f $(DOCKER_LIB_IMAGE_DIR)/$(NAME)/.version exit fi dir=$$(make _docker-get-dir) if [ -n "$(VERSION)" ]; then - echo "001---------------------------Inside set image version if $(VERSION), $$(make _docker-get-dir)" echo $(VERSION) > $$dir/.version else - echo "002---------------------------Inside set image version else $$(cat $$dir/VERSION)" echo $$(cat $$dir/VERSION) | \ sed "s/YYYY/$$(date --date=$(BUILD_DATE) -u +"%Y")/g" | \ sed "s/mm/$$(date --date=$(BUILD_DATE) -u +"%m")/g" | \ @@ -292,7 +275,6 @@ docker-image-set-version: ### Set effective Docker image version - mandatory: NA sed "s/SS/$$(date --date=$(BUILD_DATE) -u +"%S")/g" | \ sed "s/hash/$$(git rev-parse --short HEAD)/g" \ > $$dir/.version - echo "-------------------------------------dir/.version = $$(cat $$dir/.version)" fi # ============================================================================== @@ -551,7 +533,7 @@ docker-run-python: ### Run python container - mandatory: CMD; optional: SH=true, image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo python:$(DOCKER_PYTHON_VERSION)) container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo python-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) if [[ ! "$(SH)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ @@ -569,7 +551,7 @@ docker-run-python: ### Run python container - mandatory: CMD; optional: SH=true, $$image \ $(CMD) else - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ From a3975cdf5302467975632493ad605d0ed763f556 Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Fri, 25 Apr 2025 16:21:38 +0100 Subject: [PATCH 08/11] All the targets required to run locally are podman enabled --- build/automation/lib/docker.mk | 74 +++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 6b258a721..6c505f797 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -45,6 +45,7 @@ COMPOSE_HTTP_TIMEOUT := $(or $(COMPOSE_HTTP_TIMEOUT), 6000) DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) +IS_PODMAN := $(shell [ "$(DOCKER_CMD)" = "podman" ] && echo true || echo false) # ============================================================================== @@ -572,6 +573,75 @@ docker-run-python: ### Run python container - mandatory: CMD; optional: SH=true, " fi +#################################################################################################################################################### +# Enabling podman for docker-run-python target + +# Detect Docker or Podman (default to Docker if both available) +# docker-run-python: ### Run python container - works with Docker or Podman +# make docker-config > /dev/null 2>&1 +# mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} + +# lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" = true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) + +# image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo python:$(DOCKER_PYTHON_VERSION)) + +# container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo python-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date -u +"%Y%m%d%H%M%S")-$$(make secret-random LENGTH=8)) + +# # Handle env-files +# if [ "$(IS_PODMAN)" = "true" ]; then \ +# envfile1=$$(mktemp); make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)" > $$envfile1; \ +# envfile2=$$(mktemp); make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)" > $$envfile2; \ +# envfile3=$$(mktemp); make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)" > $$envfile3; \ +# envfile4=$$(mktemp); make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE) > $$envfile4; \ +# env_files="--env-file=$$envfile1 --env-file=$$envfile2 --env-file=$$envfile3 --env-file=$$envfile4"; \ +# else \ +# env_files="--env-file=<(make _list-variables PATTERN=\"^(AWS|TX|TEXAS|NHSD|TERRAFORM)\") \ +# --env-file=<(make _list-variables PATTERN=\"^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)\") \ +# --env-file=<(make _list-variables PATTERN=\"^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)\") \ +# --env-file=<(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE))"; \ +# fi; \ + +# # Choose shell or not +# if [[ ! "$(SH)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then \ +# eval $$(echo $(DOCKER_CMD)) run --interactive $(_TTY) --rm \ +# --name $$container \ +# --user $$(id -u):$$(id -g) \ +# $$env_files \ +# --env PIP_TARGET=/tmp/.packages \ +# --env PYTHONPATH=/tmp/.packages \ +# --env XDG_CACHE_HOME=/tmp/.cache \ +# --volume $(PROJECT_DIR):/project$(if $(filter podman,$(DOCKER_CMD)),:Z,) \ +# $$lib_volume_mount \ +# --network $(DOCKER_NETWORK) \ +# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ +# $(ARGS) \ +# $$image \ +# $(CMD); \ +# else \ +# eval $$(echo $(DOCKER_CMD)) run --interactive $(_TTY) --rm \ +# --name $$container \ +# --user $$(id -u):$$(id -g) \ +# $$env_files \ +# --env PIP_TARGET=/tmp/.packages \ +# --env PYTHONPATH=/tmp/.packages \ +# --env XDG_CACHE_HOME=/tmp/.cache \ +# --volume $(PROJECT_DIR):/project$(if $(filter podman,$(DOCKER_CMD)),:Z,) \ +# $$lib_volume_mount \ +# --network $(DOCKER_NETWORK) \ +# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ +# $(ARGS) \ +# $$image \ +# /bin/sh -c "$(CMD)"; \ +# fi; \ + +# # Cleanup temp env files +# if [ "$(IS_PODMAN)" = "true" ]; then \ +# rm -f $$envfile1 $$envfile2 $$envfile3 $$envfile4; \ +# fi + + +#################################################################################################################################################### + docker-run-sonar-scanner-cli: ### Run sonar-scanner-cli container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] make docker-config > /dev/null 2>&1 mkdir -p $(TMP_DIR)/.sonar/cache @@ -615,7 +685,7 @@ docker-run-terraform-tfsec: ### Run terraform tfsec container - optional: DIR,AR make docker-config > /dev/null 2>&1 image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo aquasec/tfsec:$(DOCKER_TERRAFORM_TFSEC_VERSION)) container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tfsec-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ @@ -634,7 +704,7 @@ docker-run-checkov: ### Run checkov container - optional: DIR,ARGS=[Docker args] make docker-config > /dev/null 2>&1 image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo bridgecrew/checkov:$(DOCKER_CHECKOV_VERSION)) container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo tfsec-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date --date=$$(date -u +"%Y-%m-%dT%H:%M:%S%z") -u +"%Y%m%d%H%M%S" 2> /dev/null)-$$(make secret-random LENGTH=8)) - docker run --interactive $(_TTY) --rm \ + $(DOCKER_CMD) run --interactive $(_TTY) --rm \ --name $$container \ --user $$(id -u):$$(id -g) \ --env-file <(make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)") \ From d6203efd3fd2be644b7b129530cf9a5c6747756b Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Mon, 28 Apr 2025 14:22:06 +0100 Subject: [PATCH 09/11] Making output flag dynamic to handle podman and docker --- build/automation/lib/docker.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 6c505f797..700266de2 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -46,6 +46,7 @@ DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) IS_PODMAN := $(shell [ "$(DOCKER_CMD)" = "podman" ] && echo true || echo false) +OUTPUT_OPTION := $(if $(filter false,$(IS_PODMAN)),--output type=docker) # ============================================================================== @@ -112,6 +113,7 @@ docker-build docker-image: ### Build Docker image - mandatory: NAME; optional: V --build-arg BUILD_BRANCH=$(BUILD_BRANCH) \ --build-arg BUILD_COMMIT_HASH=$(BUILD_COMMIT_HASH) \ --build-arg BUILD_COMMIT_DATE=$(BUILD_COMMIT_DATE) \ + $(OUTPUT_OPTION) \ --label name=$$IMAGE \ --label version=$$VERSION \ --label build-id=$(BUILD_ID) \ From ba4ff1636de0bdcc257f7c6c45401dd07ae07170 Mon Sep 17 00:00:00 2001 From: Matthew Begley <60427904+mabe13@users.noreply.github.com> Date: Thu, 8 May 2025 14:15:03 +0100 Subject: [PATCH 10/11] Changes to enable setting of container manager via enviroment variable Signed-off-by: Matthew Begley <60427904+mabe13@users.noreply.github.com> --- Makefile | 4 ++++ build/automation/lib/docker.mk | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b75be05c9..be1b8d23a 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ include $(abspath $(PROJECT_DIR)/build/automation/init.mk) setup: project-config # Set up project make tester-build +container-check: # Check the container management tool - can be set with: "export DOCKER_CMD=`which podman`" or "export DOCKER_CMD=`which docker`" + echo $(DOCKER_CMD) + build: # Build lambdas for IMAGE_NAME in $$(echo $(PROJECT_LAMBDAS_LIST) | tr "," "\n"); do make -s build-lambda GENERIC_IMAGE_NAME=lambda NAME=$$IMAGE_NAME @@ -556,3 +559,4 @@ python-run-ruff-fixes: # Auto fixes ruff warnings .SILENT: docker-run-ruff \ commit-date-hash-tag \ performance-test-results \ + container-check \ diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 700266de2..5127d9485 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -44,8 +44,8 @@ DOCKER_LIBRARY_TOOLS_VERSION = $(shell cat $(DOCKER_LIB_IMAGE_DIR)/tools/VERSION COMPOSE_HTTP_TIMEOUT := $(or $(COMPOSE_HTTP_TIMEOUT), 6000) DOCKER_CLIENT_TIMEOUT := $(or $(DOCKER_CLIENT_TIMEOUT), 6000) -DOCKER_CMD=$(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman) -IS_PODMAN := $(shell [ "$(DOCKER_CMD)" = "podman" ] && echo true || echo false) +DOCKER_CMD := $(or $(DOCKER_CMD), $(shell command -v docker >/dev/null 2>&1 && echo docker || echo podman)) +IS_PODMAN := $(shell [[ "$(DOCKER_CMD)" == *podman ]] && echo true || echo false) OUTPUT_OPTION := $(if $(filter false,$(IS_PODMAN)),--output type=docker) # ============================================================================== From 8617f8c37d787757660d5dc6194a31b5dde00c2a Mon Sep 17 00:00:00 2001 From: Ajay Mudgal Date: Thu, 8 May 2025 15:42:42 +0100 Subject: [PATCH 11/11] Removing commented code --- build/automation/lib/docker.mk | 69 ---------------------------------- 1 file changed, 69 deletions(-) diff --git a/build/automation/lib/docker.mk b/build/automation/lib/docker.mk index 5127d9485..07a199ba3 100644 --- a/build/automation/lib/docker.mk +++ b/build/automation/lib/docker.mk @@ -575,75 +575,6 @@ docker-run-python: ### Run python container - mandatory: CMD; optional: SH=true, " fi -#################################################################################################################################################### -# Enabling podman for docker-run-python target - -# Detect Docker or Podman (default to Docker if both available) -# docker-run-python: ### Run python container - works with Docker or Podman -# make docker-config > /dev/null 2>&1 -# mkdir -p $(TMP_DIR)/.python/pip/{cache,packages} - -# lib_volume_mount=$$(([ $(BUILD_ID) -eq 0 ] || [ "$(LIB_VOLUME_MOUNT)" = true ]) && echo "--volume $(TMP_DIR)/.python/pip/cache:/tmp/.cache/pip --volume $(TMP_DIR)/.python/pip/packages:/tmp/.packages" ||:) - -# image=$$([ -n "$(IMAGE)" ] && echo $(IMAGE) || echo python:$(DOCKER_PYTHON_VERSION)) - -# container=$$([ -n "$(CONTAINER)" ] && echo $(CONTAINER) || echo python-$(BUILD_COMMIT_HASH)-$(BUILD_ID)-$$(date -u +"%Y%m%d%H%M%S")-$$(make secret-random LENGTH=8)) - -# # Handle env-files -# if [ "$(IS_PODMAN)" = "true" ]; then \ -# envfile1=$$(mktemp); make _list-variables PATTERN="^(AWS|TX|TEXAS|NHSD|TERRAFORM)" > $$envfile1; \ -# envfile2=$$(mktemp); make _list-variables PATTERN="^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)" > $$envfile2; \ -# envfile3=$$(mktemp); make _list-variables PATTERN="^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)" > $$envfile3; \ -# envfile4=$$(mktemp); make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE) > $$envfile4; \ -# env_files="--env-file=$$envfile1 --env-file=$$envfile2 --env-file=$$envfile3 --env-file=$$envfile4"; \ -# else \ -# env_files="--env-file=<(make _list-variables PATTERN=\"^(AWS|TX|TEXAS|NHSD|TERRAFORM)\") \ -# --env-file=<(make _list-variables PATTERN=\"^(DB|DATABASE|SMTP|APP|APPLICATION|UI|API|SERVER|HOST|URL)\") \ -# --env-file=<(make _list-variables PATTERN=\"^(PROFILE|ENVIRONMENT|BUILD|PROGRAMME|ORG|SERVICE|PROJECT)\") \ -# --env-file=<(make _docker-get-variables-from-file VARS_FILE=$(VARS_FILE))"; \ -# fi; \ - -# # Choose shell or not -# if [[ ! "$(SH)" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$$ ]]; then \ -# eval $$(echo $(DOCKER_CMD)) run --interactive $(_TTY) --rm \ -# --name $$container \ -# --user $$(id -u):$$(id -g) \ -# $$env_files \ -# --env PIP_TARGET=/tmp/.packages \ -# --env PYTHONPATH=/tmp/.packages \ -# --env XDG_CACHE_HOME=/tmp/.cache \ -# --volume $(PROJECT_DIR):/project$(if $(filter podman,$(DOCKER_CMD)),:Z,) \ -# $$lib_volume_mount \ -# --network $(DOCKER_NETWORK) \ -# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ -# $(ARGS) \ -# $$image \ -# $(CMD); \ -# else \ -# eval $$(echo $(DOCKER_CMD)) run --interactive $(_TTY) --rm \ -# --name $$container \ -# --user $$(id -u):$$(id -g) \ -# $$env_files \ -# --env PIP_TARGET=/tmp/.packages \ -# --env PYTHONPATH=/tmp/.packages \ -# --env XDG_CACHE_HOME=/tmp/.cache \ -# --volume $(PROJECT_DIR):/project$(if $(filter podman,$(DOCKER_CMD)),:Z,) \ -# $$lib_volume_mount \ -# --network $(DOCKER_NETWORK) \ -# --workdir /project/$(shell echo $(abspath $(DIR)) | sed "s;$(PROJECT_DIR);;g") \ -# $(ARGS) \ -# $$image \ -# /bin/sh -c "$(CMD)"; \ -# fi; \ - -# # Cleanup temp env files -# if [ "$(IS_PODMAN)" = "true" ]; then \ -# rm -f $$envfile1 $$envfile2 $$envfile3 $$envfile4; \ -# fi - - -#################################################################################################################################################### - docker-run-sonar-scanner-cli: ### Run sonar-scanner-cli container - mandatory: CMD; optional: SH=true,DIR,ARGS=[Docker args],VARS_FILE=[Makefile vars file],IMAGE=[image name],CONTAINER=[container name] make docker-config > /dev/null 2>&1 mkdir -p $(TMP_DIR)/.sonar/cache