Merge develop => main

.dlt/config.toml

@@ -0,0 +1,19 @@
+[normalize.data_writer]
+disable_compression = "false"
+
+[runtime]
+http_show_error_body = "true"
+log_level = "INFO"
+log_format = "JSON"
+
+[extract]
+workers = 10
+
+# 'local_fs' destination: save to the `output` directory
+[destination.local_fs]
+destination_type = "filesystem"
+bucket_url = "/output_dir"
+
+[destination.minio]
+destination_type = "filesystem"
+bucket_url = "s3://cdm-lake/tenant-general-warehouse/kbase/datasets/uniprot/"

.github/workflows/docker_build.yaml

@@ -0,0 +1,59 @@
+# This is boilerplate for publishing a Docker image via Github Actions.
+
+name: Docker
+
+on:
+    workflow_dispatch:
+    push:
+        branches: ["main", "develop"]
+        # Publish semver tags as releases.
+        tags:
+            - "v[0-9]+.[0-9]+.[0-9]+"
+            - "[0-9]+.[0-9]+.[0-9]+"
+            - "[0-9]+.[0-9]+.[0-9]+-*"
+    pull_request:
+        branches: ["main", "develop"]
+    release:
+        types: [published]
+
+env:
+    REGISTRY: ghcr.io
+    IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: write
+
+        steps:
+            - name: Login to GitHub Container Registry
+              uses: docker/login-action@v3
+              with:
+                  registry: ${{ env.REGISTRY }}
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Checkout repository
+              uses: actions/checkout@v4
+
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v3
+
+            - name: Extract Docker metadata
+              id: meta
+              uses: docker/metadata-action@v5
+              with:
+                  images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+            - name: Build and push Docker image
+              id: build-and-push
+              uses: docker/build-push-action@v5
+              with:
+                  context: .
+                  push: true
+                  tags: ${{ steps.meta.outputs.tags }}
+                  labels: ${{ steps.meta.outputs.labels }}
+                  cache-from: type=gha
+                  cache-to: type=gha,mode=max

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ permissions:
 
 on:
   push:
-    branches: [main]
+    branches: [main, develop]
 
   pull_request:
     types:
@@ -87,7 +87,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.12", "3.13"]
+        python-version: ["3.13", "3.14"]
         os: ["ubuntu-24.04"]
 
     steps:
@@ -103,7 +103,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
-        run: uv sync --dev
+        run: uv sync --dev --group local
 
       - name: Run local tests
         shell: bash

.gitignore

@@ -166,5 +166,6 @@ cython_debug/
 #.idea/
 
 
-#Ignore vscode AI rules
+# codacy stuff
 .github/instructions/codacy.instructions.md
+.codacy

.vscode/settings.json

@@ -18,5 +18,6 @@
     "python.testing.unittestEnabled": false,
     "python.testing.pytestArgs": [
         "tests"
-    ]
+    ],
+    "editor.formatOnSave": true
 }

Dockerfile

@@ -0,0 +1,44 @@
+# Use a Python image with uv pre-installed
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
+
+# Install the project into `/app`
+WORKDIR /app
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Omit development dependencies
+ENV UV_NO_DEV=1
+
+# Ensure installed tools can be executed out of the box
+ENV UV_TOOL_BIN_DIR=/usr/local/bin
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --locked --no-install-project
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked
+
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
+
+# Setup a non-root user
+RUN groupadd --system --gid 999 nonroot \
+ && useradd --system --gid 999 --uid 999 --create-home nonroot
+
+# Use the non-root user to run our application
+USER nonroot
+
+# Reset the entrypoint, don't invoke `uv`
+ENTRYPOINT []
+
+# CMD ["uv", "run", "python", "--version"]

README.md

@@ -98,7 +98,7 @@ The standard python `coverage` package is used and coverage can be generated as
 
 ## Loading genomes, contigs, and features
 
-The [genome loader](src/parsers/genome_loader.py) can be used to load and integrate data from related GFF and FASTA files. Currently, the loader requires a GFF file and two FASTA files (one for amino acid seqs, one for nucleic acid seqs) for each genome. The list of files to be processed should be specified in the genome paths file, which has the following format:
+The [genome loader](src/cdm_data_loader_utils/parsers/genome_loader.py) can be used to load and integrate data from related GFF and FASTA files. Currently, the loader requires a GFF file and two FASTA files (one for amino acid seqs, one for nucleic acid seqs) for each genome. The list of files to be processed should be specified in the genome paths file, which has the following format:
 
 ```json
 {