👷 only use Python 3.10 as minimal version in CI test

.github/workflows/ci.yaml

@@ -7,8 +7,33 @@ on:
   release:
     types: [published]
   schedule:
-    - cron: '0 2 * * 3,6'
+    - cron: "0 2 * * 3,6"
+
+# runner images:
+# https://github.com/actions/runner-images
 jobs:
+  format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable
+      - uses: isort/isort-action@v1
+  lint:
+    name: Lint with ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install ruff
+        run: |
+          pip install ruff
+      - name: Lint with ruff
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          ruff check pimmslearn
   run-unit-and-integration-tests-with-conda-install:
     runs-on: ${{ matrix.os }}
     defaults:
@@ -17,66 +42,67 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest",
-             "macos-13",
-             # "windows-latest" # rrcovNA cannot be build from source on windows-server
-             ]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        os: [
+            "ubuntu-latest",
+            "macos-latest",
+            # "windows-latest" # rrcovNA cannot be build from source on windows-server
+          ]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Set up Miniconda
-      uses: conda-incubator/setup-miniconda@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-        channel-priority: strict
-        environment-file: snakemake_env.yml
-        activate-environment: snakemake
-        auto-activate-base: true
-        auto-update-conda: true
-    - name: inspect-conda-environment
-      run: |
-        conda info
-        conda list
-        conda env export --from-history --no-builds > environment.yml
-        conda env export --no-builds
-        conda env export --no-builds > environment_w_versions.yml
-    - name: Dry-Run demo workflow (integration test)
-      run: |
-       cd project
-       snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda -n
-    - name: Run demo workflow (integration test)
-      continue-on-error: true
-      run: |
-       cd project
-       snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda
-    - name: Run demo workflow again (in case of installation issues)
-      continue-on-error: true
-      run: |
-        cd project
-        snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda
-    - name: Run demo workflow again (in case of installation issues) - one thread
-      run: |
-        cd project
-        snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda
-    - name: Archive results
-    # https://github.com/actions/upload-artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ matrix.os }}-${{ matrix.python-version }}-example-workflow-results
-        path: |
-          project/runs/example/
-          snakemake_env
-          project/.snakemake/conda/
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          channel-priority: strict
+          environment-file: snakemake_env.yml
+          activate-environment: snakemake
+          auto-activate-base: true
+          auto-update-conda: true
+      - name: inspect-conda-environment
+        run: |
+          conda info
+          conda list
+          conda env export --from-history --no-builds > environment.yml
+          conda env export --no-builds
+          conda env export --no-builds > environment_w_versions.yml
+      - name: Dry-Run demo workflow (integration test)
+        run: |
+          cd project
+          snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda -n
+      - name: Run demo workflow (integration test)
+        continue-on-error: true
+        run: |
+          cd project
+          snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda
+      - name: Run demo workflow again (in case of installation issues)
+        continue-on-error: true
+        run: |
+          cd project
+          snakemake -p -c4 -k --configfile config/single_dev_dataset/example/config.yaml --use-conda
+      - name: Run demo workflow again (in case of installation issues) - one thread
+        run: |
+          cd project
+          snakemake -p -c1 --configfile config/single_dev_dataset/example/config.yaml --use-conda
+      - name: Archive results
+        # https://github.com/actions/upload-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.os }}-${{ matrix.python-version }}-example-workflow-results
+          path: |
+            project/runs/example/
+            snakemake_env
+            project/.snakemake/conda/
 
-  run-unit-local-pip-installation:
+  run-unit-tests-local-pip-installation:
     runs-on: ${{ matrix.os }}
     name: test-pip-installation
     strategy:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -118,15 +144,17 @@ jobs:
     if: startsWith(github.ref, 'refs/tags')
     needs:
       - run-unit-and-integration-tests-with-conda-install
-      - run-unit-local-pip-installation
+      - run-unit-tests-local-pip-installation
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write
 
     steps:
       - uses: actions/checkout@v4
 
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install twine and build
         run: python -m pip install --upgrade twine build
       - name: Build

environment.yml

@@ -10,7 +10,7 @@ channels:
 dependencies:
   - python>=3.9,<=3.12
   - numpy
-  - pandas>=1
+  - pandas>=1,<3.0
   - scipy>=1.6
   # plotting
   - matplotlib>=3.4
@@ -23,6 +23,7 @@ dependencies:
   # - pytorch-cuda
   - scikit-learn
   - fastai
+  - fastprogress<1.1.0
   - torchvision
   # - cudatoolkit #=11.7
   # - tensorboard

project/workflow/envs/pimms.yaml

@@ -10,7 +10,7 @@ channels:
 dependencies:
   - python>=3.9,<=3.12
   - numpy
-  - pandas>=1
+  - pandas>=1,<3.0
   - scipy>=1.6
   # plotting
   - matplotlib
@@ -23,6 +23,7 @@ dependencies:
   # - pytorch-cuda
   - scikit-learn
   - fastai
+  - fastprogress<1.1.0
   - torchvision
   # - cudatoolkit #=11.7
   # - tensorboard

project/workflow/envs/trainRmodels.yaml

@@ -19,7 +19,7 @@ dependencies:
   - r-e1071
   - r-norm
   - r-missforest
-  - r-vim
+  # - r-vim
   - r-mice
   - r-cluster
   - r-mvtnorm

pyproject.toml

@@ -16,17 +16,19 @@ classifiers = [
   "Topic :: Scientific/Engineering :: Bio-Informatics",
 ]
 dependencies = [
-  "pandas",
+  "pandas<3.0",
   "numpy",
   "torch",
   "fastai",
+  "fastprogress<1.1.0", #
+  "ipython",            #https://github.com/AnswerDotAI/fastprogress/issues/117
   "scikit-learn>=1.0",
   "seaborn",
   "matplotlib",
-  "plotly", # not used in library, but workflow
+  "plotly",             # not used in library, but workflow
   "omegaconf",
   "pingouin",
-  "njab>=0.1"
+  "njab>=0.1",
 ]
 
 [project.scripts]

snakemake_env.yml

@@ -3,5 +3,5 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-  - snakemake-minimal
+  - snakemake-minimal<9.0
   - mamba<2.0