diff --git a/.dockerignore b/.dockerignore index 5de4b142..fd683b05 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,23 +1,20 @@ */.gitignore -# Ignore node_modules directory -frontend/node_modules/ -frontend/.nx/cache/ +# Node/Nx artifacts (keep build contexts small) +**/node_modules +**/.nx -# Ignore package-lock.json or yarn.lock file -frontend/package-lock.json -frontend/yarn.lock +# Frontend build outputs and logs +services/frontend/dist +services/frontend/tmp +services/frontend/coverage +services/frontend/apps/*/dist +services/frontend/*.log -# Ignore any build artifacts or compiled files -frontend/dist/ -frontend/build/ -frontend/*.log - -# Ignore python envs and compiled python files +# Python envs and compiled python files */venv */.venv */.env */*.pyc - **/.notebooks diff --git a/.env.template b/.env.template index 4c837066..95be1696 100644 --- a/.env.template +++ b/.env.template @@ -10,7 +10,9 @@ S3_SECRET_ACCESS_KEY=your_s3_secret_key_here # ============================================================================= # Basic Authentication (Required) # ============================================================================= -BASIC_AUTH="foo:$apr1$ryE0iE7H$F2SlPDNoFdGoaHrcla2HL/" +# Used for backend/admin endpoints and reused by the frontend's auth prompt. +BASIC_AUTH_USER=foo +BASIC_AUTH_PASSWORD=bar # ============================================================================= # Langfuse Configuration (Required for observability) @@ -27,12 +29,6 @@ LANGFUSE_INIT_USER_EMAIL="user@stackit.cloud" LANGFUSE_INIT_USER_NAME="stackiteer" LANGFUSE_INIT_USER_PASSWORD="stackit123" -# ============================================================================= -# Frontend Authentication (Required) -# ============================================================================= -VITE_AUTH_USERNAME=foo -VITE_AUTH_PASSWORD=bar - # ============================================================================= # LLM Provider API Keys (Choose one or more) # ============================================================================= diff --git a/.gitignore b/.gitignore index 849a7407..db7abfcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +**/.env.langfuse **/.nx/* *.tgz */Chart.lock diff --git a/README.md b/README.md index 6171103e..9cc2a253 100644 --- a/README.md +++ b/README.md @@ -208,28 +208,33 @@ For local deployment, a few env variables need to be provided by an `.env` file The `.env` needs to contain the following values: ```dotenv -BASIC_AUTH=Zm9vOiRhcHIxJGh1VDVpL0ZKJG10elZQUm1IM29JQlBVMlZ4YkpUQy8K +# Basic auth (used by backend/admin ingress and the frontend prompt) +BASIC_AUTH_USER=... +BASIC_AUTH_PASSWORD=... S3_ACCESS_KEY_ID=... S3_SECRET_ACCESS_KEY=... -VITE_AUTH_USERNAME=... -VITE_AUTH_PASSWORD=... +LANGFUSE_PUBLIC_KEY=pk-lf-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +LANGFUSE_SECRET_KEY=sk-lf-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# Optional keys RAGAS_OPENAI_API_KEY=... - STACKIT_VLLM_API_KEY=... STACKIT_EMBEDDER_API_KEY=... -# ONLY necessary, if no init values are set. if init values are set, -# the following two values should match the init values or be commented out -# or be created via the langfuse UI. -LANGFUSE_PUBLIC_KEY=pk-lf-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -LANGFUSE_SECRET_KEY=sk-lf-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +# Optional Langfuse init values (omit if you already created a project/user) +LANGFUSE_INIT_ORG_ID=... +LANGFUSE_INIT_PROJECT_ID=... +LANGFUSE_INIT_PROJECT_PUBLIC_KEY=... +LANGFUSE_INIT_PROJECT_SECRET_KEY=... +LANGFUSE_INIT_USER_EMAIL=... +LANGFUSE_INIT_USER_NAME=... +LANGFUSE_INIT_USER_PASSWORD=... ``` -This results in a basic auth with username=`foo` and password=`bar`. +Using the defaults in `.env.template` results in a basic auth with username=`foo` and password=`bar`. > 📝 NOTE: All values containing `...` are placeholders and have to be replaced with real values. > This deployment comes with multiple options. You change the `global.config.envs.rag_class_types.RAG_CLASS_TYPE_LLM_TYPE` in the helm-deployment to on of the following values: @@ -238,6 +243,12 @@ This results in a basic auth with username=`foo` and password=`bar`. > - `ollama`: Uses ollama as an LLM provider. > +##### Langfuse init secret (dev-only, Tilt + Kustomize) + +- Copy `infrastructure/kustomize/langfuse/.env.langfuse.template` to `infrastructure/kustomize/langfuse/.env.langfuse` and fill in your Langfuse init values (org/project/user and API keys) before starting Tilt. +- Tilt runs Kustomize automatically to create a stable `langfuse-init-secrets` secret before Helm applies manifests. +- Use this helper only for local/dev. For production, manage secrets via your usual mechanism and point `secretKeyRef.name` in `values.yaml` to your precreated secrets. + #### 1.4.1 Environment Variables Setup Before running the application, you need to configure environment variables. Copy the provided example file and fill in your values: @@ -483,4 +494,3 @@ The linting-settings can be changed in the `services/rag-backend/pyproject.toml` ## 4. Contribution Guidelines In order to contribute please consult the [CONTRIBUTING.md](./CONTRIBUTING.md). - diff --git a/Tiltfile b/Tiltfile index cecbc2ac..960bc060 100644 --- a/Tiltfile +++ b/Tiltfile @@ -320,6 +320,14 @@ local_resource( allow_parallel=True, ) +# Dev-only Langfuse init secrets via Kustomize (stable name, no hash suffix) +langfuse_kustomize_dir = "./infrastructure/kustomize/langfuse" +langfuse_env_file = "%s/.env.langfuse" % langfuse_kustomize_dir +if os.path.exists(langfuse_env_file): + watch_file(langfuse_env_file) + watch_file("%s/kustomization.yaml" % langfuse_kustomize_dir) + k8s_yaml(kustomize(langfuse_kustomize_dir)) + ######################################################################################################################## ################################## build document extractor image and do live update ############################################## ######################################################################################################################## @@ -469,14 +477,13 @@ docker_build( ######################################################################################################################## value_override = [ # secrets env - "shared.secrets.s3.accessKey=%s" % os.environ["S3_ACCESS_KEY_ID"], - "shared.secrets.s3.secretKey=%s" % os.environ["S3_SECRET_ACCESS_KEY"], - "backend.secrets.basicAuth=%s" % os.environ["BASIC_AUTH"], - "backend.secrets.langfuse.publicKey=%s" % os.environ["LANGFUSE_PUBLIC_KEY"], - "backend.secrets.langfuse.secretKey=%s" % os.environ["LANGFUSE_SECRET_KEY"], - "backend.secrets.ragas.openaiApikey=%s" % os.environ["RAGAS_OPENAI_API_KEY"], - "frontend.secrets.viteAuth.VITE_AUTH_USERNAME=%s" % os.environ["VITE_AUTH_USERNAME"], - "frontend.secrets.viteAuth.VITE_AUTH_PASSWORD=%s" % os.environ["VITE_AUTH_PASSWORD"], + "shared.secrets.s3.accessKey.value=%s" % os.environ["S3_ACCESS_KEY_ID"], + "shared.secrets.s3.secretKey.value=%s" % os.environ["S3_SECRET_ACCESS_KEY"], + "shared.secrets.basicAuth.user.value=%s" % os.environ["BASIC_AUTH_USER"], + "shared.secrets.basicAuth.password.value=%s" % os.environ["BASIC_AUTH_PASSWORD"], + "backend.secrets.langfuse.publicKey.value=%s" % os.environ["LANGFUSE_PUBLIC_KEY"], + "backend.secrets.langfuse.secretKey.value=%s" % os.environ["LANGFUSE_SECRET_KEY"], + "backend.secrets.ragas.openaiApikey.value=%s" % os.environ["RAGAS_OPENAI_API_KEY"], # variables "shared.debug.backend.enabled=%s" % backend_debug, "features.frontend.enabled=true", @@ -487,21 +494,6 @@ value_override = [ "features.mcp.enabled=true", # ingress host names "backend.ingress.host.name=rag.localhost", - # langfuse - "langfuse.langfuse.additionalEnv[0].name=LANGFUSE_INIT_ORG_ID", - "langfuse.langfuse.additionalEnv[0].value=\"%s\"" % os.environ["LANGFUSE_INIT_ORG_ID"], - "langfuse.langfuse.additionalEnv[1].name=LANGFUSE_INIT_PROJECT_ID", - "langfuse.langfuse.additionalEnv[1].value=\"%s\"" % os.environ["LANGFUSE_INIT_PROJECT_ID"], - "langfuse.langfuse.additionalEnv[2].name=LANGFUSE_INIT_PROJECT_PUBLIC_KEY", - "langfuse.langfuse.additionalEnv[2].value=%s" % os.environ["LANGFUSE_INIT_PROJECT_PUBLIC_KEY"], - "langfuse.langfuse.additionalEnv[3].name=LANGFUSE_INIT_PROJECT_SECRET_KEY", - "langfuse.langfuse.additionalEnv[3].value=%s" % os.environ["LANGFUSE_INIT_PROJECT_SECRET_KEY"], - "langfuse.langfuse.additionalEnv[4].name=LANGFUSE_INIT_USER_EMAIL", - "langfuse.langfuse.additionalEnv[4].value=%s" % os.environ["LANGFUSE_INIT_USER_EMAIL"], - "langfuse.langfuse.additionalEnv[5].name=LANGFUSE_INIT_USER_PASSWORD", - "langfuse.langfuse.additionalEnv[5].value=%s" % os.environ["LANGFUSE_INIT_USER_PASSWORD"], - "langfuse.langfuse.additionalEnv[6].name=LANGFUSE_INIT_USER_NAME", - "langfuse.langfuse.additionalEnv[6].value=%s" % os.environ["LANGFUSE_INIT_USER_NAME"], ] def has_confluence_config(): @@ -532,13 +524,13 @@ if has_confluence_config(): if os.environ.get("STACKIT_VLLM_API_KEY", False): stackit_vllm_settings = [ - "backend.secrets.stackitVllm.apiKey=%s" % os.environ["STACKIT_VLLM_API_KEY"], + "backend.secrets.stackitVllm.apiKey.value=%s" % os.environ["STACKIT_VLLM_API_KEY"], ] value_override.extend(stackit_vllm_settings) if os.environ.get("STACKIT_EMBEDDER_API_KEY", False): stackit_embedder_settings = [ - "backend.secrets.stackitEmbedder.apiKey=%s" % os.environ["STACKIT_EMBEDDER_API_KEY"], + "backend.secrets.stackitEmbedder.apiKey.value=%s" % os.environ["STACKIT_EMBEDDER_API_KEY"], ] value_override.extend(stackit_embedder_settings) diff --git a/infrastructure/README.md b/infrastructure/README.md index fddffec4..f6a4e4ae 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -182,6 +182,12 @@ Default values for the deployment are provided in the `rag/values.yaml` file und > >All values containing `...` are placeholders and have to be replaced with real values. +**Dev helper via Kustomize/Tilt** +For local development you can let Tilt generate Langfuse init secrets automatically: +- Copy `infrastructure/kustomize/langfuse/.env.langfuse.template` to `infrastructure/kustomize/langfuse/.env.langfuse` and fill it with the Langfuse init env values. +- Tilt runs Kustomize on `infrastructure/kustomize/langfuse` and applies the resulting `langfuse-init-secrets` (hash disabled) before Helm resources. +- This is dev-only. For production, create/manage secrets with your secret manager and set `secretKeyRef.name` in `values.yaml` to your managed secret. + ### 1.2 Qdrant The deployment of the Qdrant can be disabled by setting the following value in the helm-chart: @@ -214,10 +220,28 @@ adminBackend: keyValueStore: USECASE_KEYVALUE_HOST: ... # Your Redis host (e.g., redis.yourdomain.com) USECASE_KEYVALUE_PORT: 6379 + secrets: + keyValueStore: + username: + value: "" # Optional inline username; prefer secretKeyRef in production + secretKeyRef: + name: "" # Existing secret containing the username + key: "USECASE_KEYVALUE_USERNAME" + password: + value: "" # Optional inline password; prefer secretKeyRef in production + secretKeyRef: + name: "" # Existing secret containing the password + key: "USECASE_KEYVALUE_PASSWORD" features: keydb: enabled: false # Disable KeyDB for production +keydb: + password: "" # Optional inline password for the bundled KeyDB chart + existingSecret: "" # Name of an existing secret that stores the KeyDB password + existingSecretPasswordKey: "password" # Key within the existing secret + auth: + username: "default" # Username that the admin backend uses when auth is enabled langfuse: valkey: @@ -233,6 +257,20 @@ langfuse: The following values should be adjusted for the deployment: ```yaml +shared: + secrets: + # Required: Basic authentication used by backend/admin ingress and frontend auth modal + basicAuthUser: + value: ... # Username for basic auth + secretKeyRef: + name: "" # Optionally reference an existing secret instead of an inline value + key: "BASIC_AUTH_USER" + basicAuthPassword: + value: ... # Password for basic auth + secretKeyRef: + name: "" + key: "BASIC_AUTH_PASSWORD" + frontend: envs: vite: @@ -246,11 +284,9 @@ frontend: host: name: ... # Your domain name (e.g., rag.yourdomain.com) - secrets: - viteAuth: - # Required: Credentials for backend authentication - VITE_AUTH_USERNAME: ... # Username for basic auth - VITE_AUTH_PASSWORD: ... # Password for basic auth +# In production, ensure a secret named "vite-auth" exists with keys +# VITE_AUTH_USERNAME and VITE_AUTH_PASSWORD set to your basic auth creds. +# (For local/dev, the chart can generate it from shared.secrets.) ``` ### 1.5 Backend @@ -262,24 +298,42 @@ The following values should be adjusted for the deployment: ```yaml backend: secrets: - # Required: Basic authentication for the backend API - basicAuth: ... # Set your basic auth credentials - + # Basic auth is configured under shared.secrets (see frontend section) # Required: Langfuse API keys for observability langfuse: - publicKey: ... # Your Langfuse public key - secretKey: ... # Your Langfuse secret key + publicKey: + value: ... # Your Langfuse public key + secretKeyRef: + name: "" # Optionally reference an existing secret instead of an inline value + key: "LANGFUSE_PUBLIC_KEY" + secretKey: + value: ... # Your Langfuse secret key + secretKeyRef: + name: "" + key: "LANGFUSE_SECRET_KEY" # Required: API keys for your chosen LLM provider # STACKIT LLM provider stackitEmbedder: - apiKey: ... # Your STACKIT embedder API key + apiKey: + value: ... # Your STACKIT embedder API key + secretKeyRef: + name: "" + key: "STACKIT_EMBEDDER_API_KEY" stackitVllm: - apiKey: ... # Your STACKIT vLLM API key + apiKey: + value: ... # Your STACKIT vLLM API key + secretKeyRef: + name: "" + key: "STACKIT_VLLM_API_KEY" # Optional: Only needed if using RAGAS evaluation with OpenAI ragas: - openaiApikey: ... # Your OpenAI API key for RAGAS evaluation + openaiApikey: + value: ... # Your OpenAI API key for RAGAS evaluation + secretKeyRef: + name: "" + key: "RAGAS_OPENAI_API_KEY" envs: # Required: Choose your LLM and embedder providers @@ -313,13 +367,15 @@ backend: ERROR_MESSAGES_NO_ANSWER_FOUND: "I'm sorry, I couldn't find an answer with the context provided." # Settings for the evaluation. You can specify the datasetname, as well as the path (in the container) where the dataset is located. langfuse: - LANGFUSE_DATASET_NAME: "test_ds" + LANGFUSE_DATASET_NAME: "rag_test_ds" LANGFUSE_DATASET_FILENAME: "/app/test_data.json" ragas: RAGAS_IS_DEBUG: false RAGAS_MODEL: "gpt-4o-mini" RAGAS_USE_OPENAI: true + RAGAS_TIMEOUT: 60 + RAGAS_EVALUATION_DATASET_NAME: "eval-data" RAGAS_MAX_CONCURRENCY: "5" ingress: diff --git a/infrastructure/kustomize/langfuse/.env.langfuse.template b/infrastructure/kustomize/langfuse/.env.langfuse.template new file mode 100644 index 00000000..6aad45a8 --- /dev/null +++ b/infrastructure/kustomize/langfuse/.env.langfuse.template @@ -0,0 +1,8 @@ +# Copy to .env.langfuse and replace with your real Langfuse init values. +LANGFUSE_INIT_ORG_ID=your-org-id +LANGFUSE_INIT_PROJECT_ID=your-project-id +LANGFUSE_INIT_PROJECT_PUBLIC_KEY=your-project-public-key +LANGFUSE_INIT_PROJECT_SECRET_KEY=your-project-secret-key +LANGFUSE_INIT_USER_EMAIL=admin@example.com +LANGFUSE_INIT_USER_NAME=Admin +LANGFUSE_INIT_USER_PASSWORD=changeme diff --git a/infrastructure/kustomize/langfuse/kustomization.yaml b/infrastructure/kustomize/langfuse/kustomization.yaml new file mode 100644 index 00000000..bbed09bf --- /dev/null +++ b/infrastructure/kustomize/langfuse/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: rag + +secretGenerator: + - name: langfuse-init-secrets + envs: + - .env.langfuse + type: Opaque + options: + # Keep a stable name so Helm/Tilt can reference the secret directly. + disableNameSuffixHash: true diff --git a/infrastructure/rag/Chart.lock b/infrastructure/rag/Chart.lock index c6d5284b..93409bfb 100644 --- a/infrastructure/rag/Chart.lock +++ b/infrastructure/rag/Chart.lock @@ -1,7 +1,7 @@ dependencies: - name: langfuse repository: https://langfuse.github.io/langfuse-k8s - version: 1.5.4 + version: 1.5.13 - name: qdrant repository: https://qdrant.github.io/qdrant-helm version: 1.15.5 @@ -14,5 +14,5 @@ dependencies: - name: ollama repository: https://otwld.github.io/ollama-helm/ version: 1.30.0 -digest: sha256:bdcbd61348805d9135cabd97c80cd08e774f479eedcbc686dc851ca55fb4dc4c -generated: "2025-10-10T08:41:01.890585511Z" +digest: sha256:398f4f0791092b57a2c0d33a715f8680fb45663499a9ad410ca4bee7eda201bb +generated: "2025-12-03T18:35:42.061121+01:00" diff --git a/infrastructure/rag/Chart.yaml b/infrastructure/rag/Chart.yaml index 619205c1..22192b6e 100644 --- a/infrastructure/rag/Chart.yaml +++ b/infrastructure/rag/Chart.yaml @@ -10,7 +10,7 @@ appVersion: "v3.4.0" dependencies: - name: langfuse repository: https://langfuse.github.io/langfuse-k8s - version: "1.5.4" + version: "1.5.13" condition: features.langfuse.enabled - name: qdrant version: 1.15.5 diff --git a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl index 0f860234..cd35eba8 100644 --- a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl +++ b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl @@ -15,6 +15,78 @@ {{- printf "%s-stackit-vllm-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- define "secret.langfuseRefName" -}} +{{- if .Values.backend.secrets.langfuse.publicKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.langfuse.publicKey.secretKeyRef.name -}} +{{- else if .Values.backend.secrets.langfuse.secretKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.langfuse.secretKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.langfuseName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.stackitVllmRefName" -}} +{{- if .Values.backend.secrets.stackitVllm.apiKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.stackitVllm.apiKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.stackitVllmName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.stackitEmbedderRefName" -}} +{{- if .Values.backend.secrets.stackitEmbedder.apiKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.stackitEmbedder.apiKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.stackitEmbedderName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.s3RefName" -}} +{{- if .Values.shared.secrets.s3.accessKey.secretKeyRef.name -}} +{{- .Values.shared.secrets.s3.accessKey.secretKeyRef.name -}} +{{- else if .Values.shared.secrets.s3.secretKey.secretKeyRef.name -}} +{{- .Values.shared.secrets.s3.secretKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.s3Name" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.keyValueStoreName" -}} +{{- printf "%s-key-value-store-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +# key value store credentials helper +{{- define "keyValueStore.credentials" -}} +{{- $result := dict "username" "" "password" "" "usernameSecretName" "" "usernameSecretKey" "" "passwordSecretName" "" "passwordSecretKey" "" -}} +{{- $defaultUsernameKey := "USECASE_KEYVALUE_USERNAME" -}} +{{- $defaultPasswordKey := "USECASE_KEYVALUE_PASSWORD" -}} + +{{- if .Values.adminBackend.secrets.keyValueStore.username.secretKeyRef.name }} +{{- $_ := set $result "usernameSecretName" .Values.adminBackend.secrets.keyValueStore.username.secretKeyRef.name -}} +{{- $_ := set $result "usernameSecretKey" (default $defaultUsernameKey .Values.adminBackend.secrets.keyValueStore.username.secretKeyRef.key) -}} +{{- else if .Values.adminBackend.secrets.keyValueStore.username.value }} +{{- $_ := set $result "username" .Values.adminBackend.secrets.keyValueStore.username.value -}} +{{- end }} + +{{- if .Values.adminBackend.secrets.keyValueStore.password.secretKeyRef.name }} +{{- $_ := set $result "passwordSecretName" .Values.adminBackend.secrets.keyValueStore.password.secretKeyRef.name -}} +{{- $_ := set $result "passwordSecretKey" (default $defaultPasswordKey .Values.adminBackend.secrets.keyValueStore.password.secretKeyRef.key) -}} +{{- else if .Values.adminBackend.secrets.keyValueStore.password.value }} +{{- $_ := set $result "password" .Values.adminBackend.secrets.keyValueStore.password.value -}} +{{- else if .Values.keydb.existingSecret }} +{{- $_ := set $result "passwordSecretName" .Values.keydb.existingSecret -}} +{{- $_ := set $result "passwordSecretKey" (default "password" .Values.keydb.existingSecretPasswordKey) -}} +{{- else if .Values.keydb.password }} +{{- $_ := set $result "password" .Values.keydb.password -}} +{{- end }} + +{{- if and (not (get $result "username")) (not (get $result "usernameSecretName")) (or (get $result "password") (get $result "passwordSecretName")) }} +{{- $_ := set $result "username" (default "default" .Values.keydb.auth.username) -}} +{{- end }} + +{{- toYaml $result -}} +{{- end -}} + # configmaps {{- define "configmap.s3Name" -}} {{- printf "%s-s3-configmap" .Release.Name | trunc 63 | trimSuffix "-" -}} diff --git a/infrastructure/rag/templates/_backend_helpers.tpl b/infrastructure/rag/templates/_backend_helpers.tpl index f1bdb14e..b186f22b 100644 --- a/infrastructure/rag/templates/_backend_helpers.tpl +++ b/infrastructure/rag/templates/_backend_helpers.tpl @@ -19,6 +19,52 @@ {{- printf "%s-stackit-ragas-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- define "secret.basicAuthName" -}} +{{- if .Values.shared.secrets.basicAuth.auth.secretKeyRef.name -}} +{{- .Values.shared.secrets.basicAuth.auth.secretKeyRef.name -}} +{{- else if .Values.shared.secrets.basicAuth.user.secretKeyRef.name -}} +{{- .Values.shared.secrets.basicAuth.user.secretKeyRef.name -}} +{{- else if .Values.shared.secrets.basicAuth.password.secretKeyRef.name -}} +{{- .Values.shared.secrets.basicAuth.password.secretKeyRef.name -}} +{{- else -}} +basic-auth +{{- end -}} +{{- end -}} + +{{- define "secret.langfuseRefName" -}} +{{- if .Values.backend.secrets.langfuse.publicKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.langfuse.publicKey.secretKeyRef.name -}} +{{- else if .Values.backend.secrets.langfuse.secretKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.langfuse.secretKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.langfuseName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.stackitEmbedderRefName" -}} +{{- if .Values.backend.secrets.stackitEmbedder.apiKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.stackitEmbedder.apiKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.stackitEmbedderName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.stackitVllmRefName" -}} +{{- if .Values.backend.secrets.stackitVllm.apiKey.secretKeyRef.name -}} +{{- .Values.backend.secrets.stackitVllm.apiKey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.stackitVllmName" . }} +{{- end -}} +{{- end -}} + +{{- define "secret.ragasRefName" -}} +{{- if .Values.backend.secrets.ragas.openaiApikey.secretKeyRef.name -}} +{{- .Values.backend.secrets.ragas.openaiApikey.secretKeyRef.name -}} +{{- else -}} +{{ template "secret.ragasName" . }} +{{- end -}} +{{- end -}} + # configmaps {{- define "configmap.publicName" -}} @@ -109,7 +155,7 @@ {{- define "backend.ingress.commonAnnotations" -}} {{- if .Values.shared.config.basicAuth.enabled }} nginx.ingress.kubernetes.io/auth-type: basic -nginx.ingress.kubernetes.io/auth-secret: basic-auth +nginx.ingress.kubernetes.io/auth-secret: {{ template "secret.basicAuthName" . }} {{- end }} nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/proxy-read-timeout: "6000" diff --git a/infrastructure/rag/templates/_helpers.tpl b/infrastructure/rag/templates/_helpers.tpl index 724bc05b..40036225 100644 --- a/infrastructure/rag/templates/_helpers.tpl +++ b/infrastructure/rag/templates/_helpers.tpl @@ -17,3 +17,60 @@ {{- define "secret.usecaseName" -}} {{- printf "%s-usecase-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* Resolve basic auth credentials from inline values or referenced secrets. */}} +{{- define "rag.basicAuthCredentials" -}} +{{- $creds := dict "username" (default "" .Values.shared.secrets.basicAuth.user.value) "password" (default "" .Values.shared.secrets.basicAuth.password.value) -}} + +{{- if and (eq (get $creds "username") "") .Values.shared.secrets.basicAuth.user.secretKeyRef.name }} + {{- with lookup "v1" "Secret" .Release.Namespace .Values.shared.secrets.basicAuth.user.secretKeyRef.name }} + {{- $key := default "BASIC_AUTH_USER" $.Values.shared.secrets.basicAuth.user.secretKeyRef.key }} + {{- $raw := index .data $key | default "" }} + {{- if ne $raw "" }} + {{- $_ := set $creds "username" (b64dec $raw) }} + {{- end }} + {{- end }} +{{- end }} + +{{- if and (eq (get $creds "password") "") .Values.shared.secrets.basicAuth.password.secretKeyRef.name }} + {{- with lookup "v1" "Secret" .Release.Namespace .Values.shared.secrets.basicAuth.password.secretKeyRef.name }} + {{- $key := default "BASIC_AUTH_PASSWORD" $.Values.shared.secrets.basicAuth.password.secretKeyRef.key }} + {{- $raw := index .data $key | default "" }} + {{- if ne $raw "" }} + {{- $_ := set $creds "password" (b64dec $raw) }} + {{- end }} + {{- end }} +{{- end }} + +{{- toYaml $creds -}} +{{- end -}} + +{{/* Build data block for the basic auth secret from htpasswd value or username/password. */}} +{{- define "rag.basicAuthSecretData" -}} +{{- $creds := (include "rag.basicAuthCredentials" . | fromYaml) -}} +{{- $auth := dict "value" (default "" .Values.shared.secrets.basicAuth.auth.value) -}} +{{- if and (eq (get $auth "value") "") .Values.shared.secrets.basicAuth.auth.secretKeyRef.name }} + {{- with lookup "v1" "Secret" .Release.Namespace .Values.shared.secrets.basicAuth.auth.secretKeyRef.name }} + {{- $key := default "auth" $.Values.shared.secrets.basicAuth.auth.secretKeyRef.key }} + {{- $raw := index .data $key | default "" }} + {{- if $raw }} + {{- $_ := set $auth "value" (b64dec $raw) }} + {{- end }} + {{- end }} +{{- end }} +{{- $providedAuth := get $auth "value" }} +{{- $fromSecretRef := or .Values.shared.secrets.basicAuth.user.secretKeyRef.name .Values.shared.secrets.basicAuth.password.secretKeyRef.name }} +{{- $data := dict -}} +{{- if $providedAuth }} + {{- $_ := set $data "auth" ($providedAuth | b64enc) }} +{{- else if and $creds.username $creds.password }} + {{- $_ := set $data "auth" (htpasswd $creds.username $creds.password | b64enc) }} + {{- if $fromSecretRef }} + {{- $_ := set $data "BASIC_AUTH_USER" ($creds.username | b64enc) }} + {{- $_ := set $data "BASIC_AUTH_PASSWORD" ($creds.password | b64enc) }} + {{- end }} +{{- end }} +{{- if gt (len $data) 0 }} +{{- toYaml $data -}} +{{- end }} +{{- end -}} diff --git a/infrastructure/rag/templates/admin-backend/deployment.yaml b/infrastructure/rag/templates/admin-backend/deployment.yaml index d86baffd..e0694d18 100644 --- a/infrastructure/rag/templates/admin-backend/deployment.yaml +++ b/infrastructure/rag/templates/admin-backend/deployment.yaml @@ -122,15 +122,42 @@ spec: - configMapRef: name: {{ template "configmap.chunkerName" . }} - secretRef: - name: {{ template "secret.langfuseName" . }} + name: {{ template "secret.langfuseRefName" . }} - secretRef: name: {{ template "secret.usecaseName" . }} - secretRef: - name: {{ template "secret.s3Name" . }} + name: {{ template "secret.s3RefName" . }} - secretRef: - name: {{ template "secret.stackitVllmName" . }} + name: {{ template "secret.stackitVllmRefName" . }} - secretRef: - name: {{ template "secret.stackitEmbedderName" . }} + name: {{ template "secret.stackitEmbedderRefName" . }} + {{- $kvCreds := (include "keyValueStore.credentials" . | fromYaml) }} env: - name: PYTHONPATH value: {{ .Values.adminBackend.pythonPathEnv.PYTHONPATH }} + {{- if $kvCreds.usernameSecretName }} + - name: USECASE_KEYVALUE_USERNAME + valueFrom: + secretKeyRef: + name: {{ $kvCreds.usernameSecretName }} + key: {{ $kvCreds.usernameSecretKey }} + {{- else if $kvCreds.username }} + - name: USECASE_KEYVALUE_USERNAME + valueFrom: + secretKeyRef: + name: {{ template "secret.keyValueStoreName" . }} + key: USECASE_KEYVALUE_USERNAME + {{- end }} + {{- if $kvCreds.passwordSecretName }} + - name: USECASE_KEYVALUE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $kvCreds.passwordSecretName }} + key: {{ $kvCreds.passwordSecretKey }} + {{- else if $kvCreds.password }} + - name: USECASE_KEYVALUE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "secret.keyValueStoreName" . }} + key: USECASE_KEYVALUE_PASSWORD + {{- end }} diff --git a/infrastructure/rag/templates/admin-backend/secrets.yaml b/infrastructure/rag/templates/admin-backend/secrets.yaml new file mode 100644 index 00000000..0df66533 --- /dev/null +++ b/infrastructure/rag/templates/admin-backend/secrets.yaml @@ -0,0 +1,15 @@ +{{- $kvCreds := (include "keyValueStore.credentials" . | fromYaml) -}} +{{- if or $kvCreds.username $kvCreds.password }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "secret.keyValueStoreName" . }} +type: Opaque +data: + {{- if $kvCreds.username }} + USECASE_KEYVALUE_USERNAME: {{ $kvCreds.username | b64enc }} + {{- end }} + {{- if $kvCreds.password }} + USECASE_KEYVALUE_PASSWORD: {{ $kvCreds.password | b64enc }} + {{- end }} +{{- end }} diff --git a/infrastructure/rag/templates/backend/deployment.yaml b/infrastructure/rag/templates/backend/deployment.yaml index c3dfbf44..5a89b19a 100644 --- a/infrastructure/rag/templates/backend/deployment.yaml +++ b/infrastructure/rag/templates/backend/deployment.yaml @@ -134,15 +134,15 @@ spec: - configMapRef: name: {{ template "configmap.retryDecoratorName" . }} - secretRef: - name: {{ template "secret.langfuseName" . }} + name: {{ template "secret.langfuseRefName" . }} - secretRef: - name: {{ template "secret.stackitVllmName" . }} + name: {{ template "secret.stackitVllmRefName" . }} - secretRef: name: {{ template "secret.usecaseName" . }} - secretRef: - name: {{ template "secret.stackitEmbedderName" . }} + name: {{ template "secret.stackitEmbedderRefName" . }} - secretRef: - name: {{ template "secret.ragasName" . }} + name: {{ template "secret.ragasRefName" . }} env: - name: PYTHONPATH value: {{ .Values.backend.pythonPathEnv.PYTHONPATH }} diff --git a/infrastructure/rag/templates/backend/secrets.yaml b/infrastructure/rag/templates/backend/secrets.yaml index c4828b67..eee03c18 100644 --- a/infrastructure/rag/templates/backend/secrets.yaml +++ b/infrastructure/rag/templates/backend/secrets.yaml @@ -1,43 +1,51 @@ +{{- $basicAuthData := include "rag.basicAuthSecretData" . }} +{{- if $basicAuthData }} apiVersion: v1 data: - auth: {{ .Values.backend.secrets.basicAuth | b64enc }} - {{- range $key, $value := .Values.backend.secrets.auth }} - {{ $key }}: {{ $value | b64enc }} - {{- end }} +{{ $basicAuthData | nindent 2 }} kind: Secret metadata: - name: basic-auth + name: {{ template "secret.basicAuthName" . }} type: Opaque +{{- end }} --- +{{- if and .Values.backend.secrets.langfuse.publicKey.value .Values.backend.secrets.langfuse.secretKey.value }} apiVersion: v1 kind: Secret metadata: name: {{ template "secret.langfuseName" . }} type: Opaque data: - LANGFUSE_PUBLIC_KEY: {{ .Values.backend.secrets.langfuse.publicKey | b64enc }} - LANGFUSE_SECRET_KEY: {{ .Values.backend.secrets.langfuse.secretKey | b64enc }} + LANGFUSE_PUBLIC_KEY: {{ .Values.backend.secrets.langfuse.publicKey.value | b64enc }} + LANGFUSE_SECRET_KEY: {{ .Values.backend.secrets.langfuse.secretKey.value | b64enc }} +{{- end }} --- +{{- if .Values.backend.secrets.stackitVllm.apiKey.value }} apiVersion: v1 kind: Secret metadata: name: {{ template "secret.stackitVllmName" . }} type: Opaque data: - STACKIT_VLLM_API_KEY: {{ .Values.backend.secrets.stackitVllm.apiKey | b64enc }} + STACKIT_VLLM_API_KEY: {{ .Values.backend.secrets.stackitVllm.apiKey.value | b64enc }} +{{- end }} --- +{{- if .Values.backend.secrets.stackitEmbedder.apiKey.value }} apiVersion: v1 kind: Secret metadata: name: {{ template "secret.stackitEmbedderName" . }} type: Opaque data: - STACKIT_EMBEDDER_API_KEY: {{ .Values.backend.secrets.stackitEmbedder.apiKey | b64enc }} + STACKIT_EMBEDDER_API_KEY: {{ .Values.backend.secrets.stackitEmbedder.apiKey.value | b64enc }} +{{- end }} --- +{{- if .Values.backend.secrets.ragas.openaiApikey.value }} apiVersion: v1 kind: Secret metadata: name: {{ template "secret.ragasName" . }} type: Opaque data: - RAGAS_OPENAI_API_KEY: {{ .Values.backend.secrets.ragas.openaiApikey | b64enc }} + RAGAS_OPENAI_API_KEY: {{ .Values.backend.secrets.ragas.openaiApikey.value | b64enc }} +{{- end }} diff --git a/infrastructure/rag/templates/extractor/deployment.yaml b/infrastructure/rag/templates/extractor/deployment.yaml index 7b5e16de..d51aaf88 100644 --- a/infrastructure/rag/templates/extractor/deployment.yaml +++ b/infrastructure/rag/templates/extractor/deployment.yaml @@ -113,7 +113,7 @@ spec: - configMapRef: name: {{ template "configmap.extractorSitemapName" . }} - secretRef: - name: {{ template "secret.s3Name" . }} + name: {{ template "secret.s3RefName" . }} {{- $hfCacheDir := include "extractor.huggingfaceCacheDir" . }} env: - name: PYTHONPATH diff --git a/infrastructure/rag/templates/frontend/secrets.yaml b/infrastructure/rag/templates/frontend/secrets.yaml index 416614a9..36637060 100644 --- a/infrastructure/rag/templates/frontend/secrets.yaml +++ b/infrastructure/rag/templates/frontend/secrets.yaml @@ -1,9 +1,9 @@ -{{- if $.Values.features.frontend.enabled }} +{{- $basicAuth := (include "rag.basicAuthCredentials" . | fromYaml) }} +{{- if and $.Values.features.frontend.enabled $basicAuth.username $basicAuth.password }} apiVersion: v1 data: - {{- range $key, $value := .Values.frontend.secrets.viteAuth }} - {{ $key }}: {{ $value | b64enc }} - {{- end }} + VITE_AUTH_USERNAME: {{ $basicAuth.username | b64enc }} + VITE_AUTH_PASSWORD: {{ $basicAuth.password | b64enc }} kind: Secret metadata: name: vite-auth diff --git a/infrastructure/rag/templates/secrets.yaml b/infrastructure/rag/templates/secrets.yaml index e85957fd..8c4410a7 100644 --- a/infrastructure/rag/templates/secrets.yaml +++ b/infrastructure/rag/templates/secrets.yaml @@ -1,11 +1,13 @@ +{{- if and .Values.shared.secrets.s3.accessKey.value .Values.shared.secrets.s3.secretKey.value }} apiVersion: v1 kind: Secret metadata: name: {{ template "secret.s3Name" . }} type: Opaque data: - S3_ACCESS_KEY_ID: {{ .Values.shared.secrets.s3.accessKey | b64enc }} - S3_SECRET_ACCESS_KEY: {{ .Values.shared.secrets.s3.secretKey | b64enc }} + S3_ACCESS_KEY_ID: {{ .Values.shared.secrets.s3.accessKey.value | b64enc }} + S3_SECRET_ACCESS_KEY: {{ .Values.shared.secrets.s3.secretKey.value | b64enc }} +{{- end }} --- apiVersion: v1 kind: Secret diff --git a/infrastructure/rag/values.yaml b/infrastructure/rag/values.yaml index 636426e9..d07679df 100644 --- a/infrastructure/rag/values.yaml +++ b/infrastructure/rag/values.yaml @@ -147,16 +147,35 @@ backend: port: 8080 secrets: - basicAuth: "" langfuse: - publicKey: "pk-lf" - secretKey: "sk-lf" + publicKey: + value: "pk-lf" + secretKeyRef: + name: "" + key: "LANGFUSE_PUBLIC_KEY" + secretKey: + value: "sk-lf" + secretKeyRef: + name: "" + key: "LANGFUSE_SECRET_KEY" stackitEmbedder: - apiKey: "" + apiKey: + value: "" + secretKeyRef: + name: "" + key: "STACKIT_EMBEDDER_API_KEY" stackitVllm: - apiKey: "sk-123" + apiKey: + value: "sk-123" + secretKeyRef: + name: "" + key: "STACKIT_VLLM_API_KEY" ragas: - openaiApikey: "" + openaiApikey: + value: "" + secretKeyRef: + name: "" + key: "RAGAS_OPENAI_API_KEY" envs: stackitVllm: @@ -249,11 +268,6 @@ frontend: pathType: ImplementationSpecific port: 8080 - secrets: - viteAuth: - VITE_AUTH_USERNAME: "" - VITE_AUTH_PASSWORD: "" - envs: vite: VITE_CHAT_AUTH_ENABLED: true @@ -326,8 +340,18 @@ adminBackend: pathType: ImplementationSpecific port: 8080 - minio: - enabled: true + secrets: + keyValueStore: + username: + value: "" + secretKeyRef: + name: "" + key: "USECASE_KEYVALUE_USERNAME" + password: + value: "" + secretKeyRef: + name: "" + key: "USECASE_KEYVALUE_PASSWORD" envs: summarizer: @@ -357,6 +381,12 @@ adminBackend: keyValueStore: USECASE_KEYVALUE_PORT: 6379 USECASE_KEYVALUE_HOST: "rag-keydb" + USECASE_KEYVALUE_USE_SSL: false + USECASE_KEYVALUE_SSL_CERT_REQS: "" + USECASE_KEYVALUE_SSL_CA_CERTS: "" + USECASE_KEYVALUE_SSL_CERTFILE: "" + USECASE_KEYVALUE_SSL_KEYFILE: "" + USECASE_KEYVALUE_SSL_CHECK_HOSTNAME: true sourceUploader: # Large sitemap ingestions (per-page summaries) can take > 1 hour. SOURCE_UPLOADER_TIMEOUT: 3600 @@ -474,10 +504,34 @@ shared: issuerKind: ClusterIssuer secrets: + basicAuth: + auth: + value: "" # Optional: precomputed htpasswd line (e.g., "user:$apr1$..."); overrides inline user/pass hashing. + secretKeyRef: + name: "" + key: "auth" + user: + value: "" + secretKeyRef: + name: "" + key: "BASIC_AUTH_USER" + password: + value: "" + secretKeyRef: + name: "" + key: "BASIC_AUTH_PASSWORD" s3: - accessKey: "admin" - secretKey: "adminpassword" - usecase: + accessKey: + value: "admin" + secretKeyRef: + name: "" + key: "S3_ACCESS_KEY_ID" + secretKey: + value: "adminpassword" + secretKeyRef: + name: "" + key: "S3_SECRET_ACCESS_KEY" + usecase: {} envs: @@ -501,6 +555,9 @@ langfuse: # Used to hash API keys salt: value: "changeme" + secretKeyRef: + name: "" + key: "" # Authentication settings features: @@ -527,23 +584,47 @@ langfuse: url: http://localhost:3000 secret: value: "changeme" + secretKeyRef: + name: "" + key: "" # Additional environment variables (only for init values) additionalEnv: - name: LANGFUSE_INIT_ORG_ID - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_ORG_ID" - name: LANGFUSE_INIT_PROJECT_ID - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_PROJECT_ID" - name: LANGFUSE_INIT_PROJECT_PUBLIC_KEY - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_PROJECT_PUBLIC_KEY" - name: LANGFUSE_INIT_PROJECT_SECRET_KEY - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_PROJECT_SECRET_KEY" - name: LANGFUSE_INIT_USER_EMAIL - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_USER_EMAIL" - name: LANGFUSE_INIT_USER_NAME - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_USER_NAME" - name: LANGFUSE_INIT_USER_PASSWORD - value: "" + valueFrom: + secretKeyRef: + name: "langfuse-init-secrets" + key: "LANGFUSE_INIT_USER_PASSWORD" # Additional init containers extraInitContainers: @@ -553,12 +634,13 @@ langfuse: - sh - -c - | - until nc -z rag-postgresql 5432; do - echo "Waiting for PostgreSQL to be ready..." - sleep 2 - done - # Define a reasonable timeout in case PostgreSQL fails to come up - timeoutSeconds: 300 + timeout 300 sh -c ' + until nc -z f17fdc4e-c4da-4cc5-ba3a-0fbe63303957.postgresql.eu01.onstackit.cloud 5432; do + echo "Waiting for PostgreSQL to be ready..." + sleep 2 + done + ' + # Stop waiting after 5 minutes if PostgreSQL never comes up. # PostgreSQL Configuration (use external PostgreSQL) postgresql: @@ -569,6 +651,9 @@ langfuse: username: postgres password: postgres database: langfuse + existingSecret: "" # NOTE: for production use existing secret to fetch password + secretKeys: + userPasswordKey: password # Redis Configuration (external KeyDB) redis: @@ -578,6 +663,8 @@ langfuse: auth: username: "default" password: "" + existingSecret: "" # NOTE: for production use existing secret to fetch password + existingSecretPasswordKey: "" # ClickHouse Configuration (external ClickHouse) clickhouse: @@ -588,6 +675,8 @@ langfuse: auth: username: "default" password: "changeme" + existingSecret: "" # NOTE: for production use existing secret to fetch password + existingSecretKey: "" migration: url: "clickhouse://rag-clickhouse:9000" ssl: false @@ -618,8 +707,14 @@ langfuse: forcePathStyle: true accessKeyId: value: "admin" + secretKeyRef: + name: "" + key: "" secretAccessKey: value: "adminpassword" + secretKeyRef: + name: "" + key: "" eventUpload: enabled: true bucket: "langfuse" @@ -628,8 +723,14 @@ langfuse: forcePathStyle: true accessKeyId: value: "admin" + secretKeyRef: + name: "" + key: "" secretAccessKey: value: "adminpassword" + secretKeyRef: + name: "" + key: "" minio: image: @@ -670,3 +771,9 @@ keydb: multiMaster: "no" activeReplicas: "no" nodes: 1 + # Authentication for the bundled KeyDB chart and the admin backend client. + password: "" # Inline password for development. Leave empty to disable auth or prefer existingSecret for production. + existingSecret: "" # Optional existing secret containing the KeyDB password. + existingSecretPasswordKey: "password" + auth: + username: "default" diff --git a/libs/admin-api-lib/README.md b/libs/admin-api-lib/README.md index 3c114ae6..45876d9f 100644 --- a/libs/admin-api-lib/README.md +++ b/libs/admin-api-lib/README.md @@ -67,6 +67,8 @@ All settings are powered by `pydantic-settings`, so you can use environment vari - `SUMMARIZER_MAXIMUM_INPUT_SIZE`, `SUMMARIZER_MAXIMUM_CONCURRENCY`, `SUMMARIZER_MAX_RETRIES`, etc. – tune summariser limits and retry behaviour. - `SOURCE_UPLOADER_TIMEOUT` – adjust how long non-file source ingestions wait before timing out. - `USECASE_KEYVALUE_HOST` / `USECASE_KEYVALUE_PORT` – configure the KeyDB/Redis instance that persists document status. +- `USECASE_KEYVALUE_USERNAME` / `USECASE_KEYVALUE_PASSWORD` – optional credentials for authenticating against KeyDB/Redis. +- `USECASE_KEYVALUE_USE_SSL`, `USECASE_KEYVALUE_SSL_CERT_REQS`, `USECASE_KEYVALUE_SSL_CA_CERTS`, `USECASE_KEYVALUE_SSL_CERTFILE`, `USECASE_KEYVALUE_SSL_KEYFILE`, `USECASE_KEYVALUE_SSL_CHECK_HOSTNAME` – optional TLS settings for managed Redis deployments (e.g., STACKIT Redis or other SSL-only endpoints). The Helm chart forwards these values through `adminBackend.envs.*`, keeping deployments declarative. Local development can rely on `.env` as described in the repository root README. diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py b/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py index ef27b65a..5eebde3a 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py @@ -1,6 +1,8 @@ """Module containing the FileStatusKeyValueStore class.""" import json +import ssl +from typing import Any from redis import Redis @@ -37,9 +39,53 @@ def __init__(self, settings: KeyValueSettings): Parameters ---------- settings : KeyValueSettings - The settings object containing the host and port information for the Redis connection. + The settings object containing the connection information for the Redis connection. """ - self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True) + redis_kwargs: dict[str, Any] = { + "host": settings.host, + "port": settings.port, + "decode_responses": True, + **self._build_ssl_kwargs(settings), + } + if settings.username: + redis_kwargs["username"] = settings.username + if settings.password: + redis_kwargs["password"] = settings.password + + self._redis = Redis(**redis_kwargs) + + @staticmethod + def _build_ssl_kwargs(settings: KeyValueSettings) -> dict[str, Any]: + """Build Redis SSL settings from configuration, mapping string values to ssl constants.""" + if not settings.use_ssl: + return {} + + cert_reqs_map = { + "required": ssl.CERT_REQUIRED, + "optional": ssl.CERT_OPTIONAL, + "none": ssl.CERT_NONE, + "cert_required": ssl.CERT_REQUIRED, + "cert_optional": ssl.CERT_OPTIONAL, + "cert_none": ssl.CERT_NONE, + } + ssl_cert_reqs = None + if settings.ssl_cert_reqs: + ssl_cert_reqs = cert_reqs_map.get(settings.ssl_cert_reqs.lower(), settings.ssl_cert_reqs) + + ssl_kwargs: dict[str, Any] = { + "ssl": True, + "ssl_check_hostname": settings.ssl_check_hostname, + } + if ssl_cert_reqs is not None: + ssl_kwargs["ssl_cert_reqs"] = ssl_cert_reqs + if settings.ssl_ca_certs: + ssl_kwargs["ssl_ca_certs"] = settings.ssl_ca_certs + if settings.ssl_certfile: + ssl_kwargs["ssl_certfile"] = settings.ssl_certfile + if settings.ssl_keyfile: + ssl_kwargs["ssl_keyfile"] = settings.ssl_keyfile + + return ssl_kwargs @staticmethod def _to_str(file_name: str, file_status: Status) -> str: diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py index 5acc6371..b992ce3a 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py @@ -14,6 +14,22 @@ class KeyValueSettings(BaseSettings): The hostname of the key value store. port : int The port number of the key value store. + username : str | None + Optional username for authenticating with the key value store. + password : str | None + Optional password for authenticating with the key value store. + use_ssl : bool + Whether to use SSL/TLS when connecting to the key value store. + ssl_cert_reqs : str | None + SSL certificate requirement level (e.g., 'required', 'optional', 'none'). + ssl_ca_certs : str | None + Path to a CA bundle file for verifying the server certificate. + ssl_certfile : str | None + Path to the client SSL certificate file (if mutual TLS is required). + ssl_keyfile : str | None + Path to the client SSL private key file (if mutual TLS is required). + ssl_check_hostname : bool + Whether to verify the server hostname against the certificate. """ class Config: @@ -24,3 +40,11 @@ class Config: host: str = Field() port: int = Field() + username: str | None = Field(default=None) + password: str | None = Field(default=None) + use_ssl: bool = Field(default=False) + ssl_cert_reqs: str | None = Field(default=None) + ssl_ca_certs: str | None = Field(default=None) + ssl_certfile: str | None = Field(default=None) + ssl_keyfile: str | None = Field(default=None) + ssl_check_hostname: bool = Field(default=True)