From 0bdb056c046cad0142a7456e53efa7d9aeae3f1c Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 24 Sep 2025 20:44:53 +0200 Subject: [PATCH 01/11] =?UTF-8?q?=F0=9F=91=B7=20only=20use=20Python=203.10?= =?UTF-8?q?=20as=20minimal=20version=20in=20CI=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d76536ba..96639852f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,7 @@ jobs: "macos-13", # "windows-latest" # rrcovNA cannot be build from source on windows-server ] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout uses: actions/checkout@v4 From dc5b3733fe674a6992e2f4abb865eb6442c57e39 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 24 Sep 2025 21:00:30 +0200 Subject: [PATCH 02/11] :bug: update images to use latest tag, remove one more 3.9 --- .github/workflows/ci.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 96639852f..c6a8f9763 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,6 +8,9 @@ on: types: [published] schedule: - cron: '0 2 * * 3,6' + +# runner images: +# https://github.com/actions/runner-images jobs: run-unit-and-integration-tests-with-conda-install: runs-on: ${{ matrix.os }} @@ -18,7 +21,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", - "macos-13", + "macos-latest", # "windows-latest" # rrcovNA cannot be build from source on windows-server ] python-version: ["3.10", "3.11", "3.12"] @@ -76,7 +79,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 with: From 7156b6ddefe5e6a8740665edc399ffb1ff17fa40 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:21:00 +0100 Subject: [PATCH 03/11] :wrench: add black, isort and black - will be done in separate PRs --- .github/workflows/ci.yaml | 129 ++++++++++++++++++++++---------------- 1 file changed, 76 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6a8f9763..6f49e3195 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,11 +7,33 @@ on: release: types: [published] schedule: - - cron: '0 2 * * 3,6' + - cron: "0 2 * * 3,6" # runner images: # https://github.com/actions/runner-images jobs: + format: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable + - uses: isort/isort-action@v1 + lint: + name: Lint with ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install ruff + run: | + pip install ruff + - name: Lint with ruff + run: | + # stop the build if there are Python syntax errors or undefined names + ruff check src run-unit-and-integration-tests-with-conda-install: runs-on: ${{ matrix.os }} defaults: @@ -20,59 +42,60 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", - "macos-latest", - # "windows-latest" # rrcovNA cannot be build from source on windows-server - ] + os: [ + "ubuntu-latest", + "macos-latest", + # "windows-latest" # rrcovNA cannot be build from source on windows-server + ] python-version: ["3.10", "3.11", "3.12"] steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Miniconda - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ matrix.python-version }} - channel-priority: strict - environment-file: snakemake_env.yml - activate-environment: snakemake - auto-activate-base: true - auto-update-conda: true - - name: inspect-conda-environment - run: | - conda info - conda list - conda env export --from-history --no-builds > environment.yml - conda env export --no-builds - conda env export --no-builds > environment_w_versions.yml - - name: Dry-Run demo workflow (integration test) - run: | - cd project - snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda -n - - name: Run demo workflow (integration test) - continue-on-error: true - run: | - cd project - snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda - - name: Run demo workflow again (in case of installation issues) - continue-on-error: true - run: | - cd project - snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda - - name: Run demo workflow again (in case of installation issues) - one thread - run: | - cd project - snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda - - name: Archive results - # https://github.com/actions/upload-artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.os }}-${{ matrix.python-version }}-example-workflow-results - path: | - project/runs/example/ - snakemake_env - project/.snakemake/conda/ + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + channel-priority: strict + environment-file: snakemake_env.yml + activate-environment: snakemake + auto-activate-base: true + auto-update-conda: true + - name: inspect-conda-environment + run: | + conda info + conda list + conda env export --from-history --no-builds > environment.yml + conda env export --no-builds + conda env export --no-builds > environment_w_versions.yml + - name: Dry-Run demo workflow (integration test) + run: | + cd project + snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda -n + - name: Run demo workflow (integration test) + continue-on-error: true + run: | + cd project + snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda + - name: Run demo workflow again (in case of installation issues) + continue-on-error: true + run: | + cd project + snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda + - name: Run demo workflow again (in case of installation issues) - one thread + run: | + cd project + snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda + - name: Archive results + # https://github.com/actions/upload-artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.python-version }}-example-workflow-results + path: | + project/runs/example/ + snakemake_env + project/.snakemake/conda/ - run-unit-local-pip-installation: + run-unit-tests-local-pip-installation: runs-on: ${{ matrix.os }} name: test-pip-installation strategy: @@ -121,7 +144,7 @@ jobs: if: startsWith(github.ref, 'refs/tags') needs: - run-unit-and-integration-tests-with-conda-install - - run-unit-local-pip-installation + - run-unit-tests-local-pip-installation runs-on: ubuntu-latest steps: @@ -129,7 +152,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.10" - name: Install twine and build run: python -m pip install --upgrade twine build - name: Build From 42bd71b9707e836820b5d505cefeed59e63be33f Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:27:42 +0100 Subject: [PATCH 04/11] :wrench: add write permission for pypi update - move away from token auth --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6f49e3195..376a1da03 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -146,6 +146,8 @@ jobs: - run-unit-and-integration-tests-with-conda-install - run-unit-tests-local-pip-installation runs-on: ubuntu-latest + permissions: + id-token: write steps: - uses: actions/checkout@v4 From bf43e0a3ae46235b053d13352373a9ccf6328c75 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:29:34 +0100 Subject: [PATCH 05/11] :bug: pimmslearn does not use src layout (yet) --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 376a1da03..bd1f7fdc5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,7 +33,7 @@ jobs: - name: Lint with ruff run: | # stop the build if there are Python syntax errors or undefined names - ruff check src + ruff check pimmslearn run-unit-and-integration-tests-with-conda-install: runs-on: ${{ matrix.os }} defaults: From f3321a95669749aad907ff77ad76a75aa842d9a2 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:33:01 +0100 Subject: [PATCH 06/11] :wrench: use snakemake v8 --- snakemake_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snakemake_env.yml b/snakemake_env.yml index 931c9cbda..ce7136dec 100644 --- a/snakemake_env.yml +++ b/snakemake_env.yml @@ -3,5 +3,5 @@ channels: - conda-forge - bioconda dependencies: - - snakemake-minimal + - snakemake-minimal<9.0 - mamba<2.0 From b1177aa6e0434e9d9bed404b5f2878abf913a6b9 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:40:27 +0100 Subject: [PATCH 07/11] :bug: fastprogress needs IPyhton - but does not install it, so add it manually. --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 73bcb27f8..aaca4bbca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,13 +20,14 @@ dependencies = [ "numpy", "torch", "fastai", + "ipython", #https://github.com/AnswerDotAI/fastprogress/issues/117 "scikit-learn>=1.0", "seaborn", "matplotlib", - "plotly", # not used in library, but workflow + "plotly", # not used in library, but workflow "omegaconf", "pingouin", - "njab>=0.1" + "njab>=0.1", ] [project.scripts] From 93b7bd4a5624cc14d496dd23a0af1bea0c5c0766 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:44:27 +0100 Subject: [PATCH 08/11] :bug: conda-forge does not have r-vim? --- project/workflow/envs/trainRmodels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/workflow/envs/trainRmodels.yaml b/project/workflow/envs/trainRmodels.yaml index c4ad06ee8..2e0cb65f1 100644 --- a/project/workflow/envs/trainRmodels.yaml +++ b/project/workflow/envs/trainRmodels.yaml @@ -19,7 +19,7 @@ dependencies: - r-e1071 - r-norm - r-missforest - - r-vim + # - r-vim - r-mice - r-cluster - r-mvtnorm From 2a7685818529a26fa821c2ac2c1c6992112fb5d7 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 17:46:45 +0100 Subject: [PATCH 09/11] :wrench: restrict to pandas 2 for now --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aaca4bbca..ff72bafcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Bio-Informatics", ] dependencies = [ - "pandas", + "pandas<3.0", "numpy", "torch", "fastai", From 56ab2d2a153a5f36462908bc2a1a10b1403194b6 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 18:20:38 +0100 Subject: [PATCH 10/11] :bug: restrict to pandas 2 --- environment.yml | 2 +- project/workflow/envs/pimms.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 729a5cacd..5bf2861b8 100644 --- a/environment.yml +++ b/environment.yml @@ -10,7 +10,7 @@ channels: dependencies: - python>=3.9,<=3.12 - numpy - - pandas>=1 + - pandas>=1,<3.0 - scipy>=1.6 # plotting - matplotlib>=3.4 diff --git a/project/workflow/envs/pimms.yaml b/project/workflow/envs/pimms.yaml index bbd10437d..f3ed0ad73 100644 --- a/project/workflow/envs/pimms.yaml +++ b/project/workflow/envs/pimms.yaml @@ -10,7 +10,7 @@ channels: dependencies: - python>=3.9,<=3.12 - numpy - - pandas>=1 + - pandas>=1,<3.0 - scipy>=1.6 # plotting - matplotlib From 306cc2a713af4517ea19e9aaa547fc0fe9226aea Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Sun, 15 Feb 2026 18:56:48 +0100 Subject: [PATCH 11/11] :wrench: restrict fastprogress --- environment.yml | 1 + project/workflow/envs/pimms.yaml | 1 + pyproject.toml | 5 +++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index 5bf2861b8..c9b67f569 100644 --- a/environment.yml +++ b/environment.yml @@ -23,6 +23,7 @@ dependencies: # - pytorch-cuda - scikit-learn - fastai + - fastprogress<1.1.0 - torchvision # - cudatoolkit #=11.7 # - tensorboard diff --git a/project/workflow/envs/pimms.yaml b/project/workflow/envs/pimms.yaml index f3ed0ad73..da9a6f386 100644 --- a/project/workflow/envs/pimms.yaml +++ b/project/workflow/envs/pimms.yaml @@ -23,6 +23,7 @@ dependencies: # - pytorch-cuda - scikit-learn - fastai + - fastprogress<1.1.0 - torchvision # - cudatoolkit #=11.7 # - tensorboard diff --git a/pyproject.toml b/pyproject.toml index ff72bafcf..7dc9a7641 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,12 @@ dependencies = [ "numpy", "torch", "fastai", - "ipython", #https://github.com/AnswerDotAI/fastprogress/issues/117 + "fastprogress<1.1.0", # + "ipython", #https://github.com/AnswerDotAI/fastprogress/issues/117 "scikit-learn>=1.0", "seaborn", "matplotlib", - "plotly", # not used in library, but workflow + "plotly", # not used in library, but workflow "omegaconf", "pingouin", "njab>=0.1",