v2.0.0-beta3

.gitignore

@@ -59,6 +59,7 @@ scripts/keycloak/data/*
 # ignore clowder chart deps
 deployments/kubernetes/charts/clowder2/charts
 deployments/kubernetes/charts/clowder2/*clowder2-software-dev.yaml
+*secret*.yaml
 
 # Environments
 .env

.run/uvicorn.run.xml

@@ -1,7 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="uvicorn" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
     <module name="clowder2" />
-    <option name="ENV_FILES" value="" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
@@ -14,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="uvicorn" />
-    <option name="PARAMETERS" value="app.main:app --reload --host 0.0.0.0" />
+    <option name="PARAMETERS" value="app.main:app --host 0.0.0.0 --workers 17" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="true" />

CHANGELOG.md

@@ -5,6 +5,34 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v2.0.0-beta.3] - 2024-07-29
+
+### Added
+
+- License management
+- Release dataset with versions
+- Enable and disable user account through Keycloak
+- Jupyterhub integration
+- Interface for creating and editing matching criteria for triggering extractors
+- Interface for editing metadata definitions
+- My dataset tab listing all the datasets created by the user
+- Drag and drop upload multiple files
+- Footer with links to documentation, source code, and contact information
+- Documentation through MKDocs
+
+### Changed
+
+- Allow public datasets and files to be searchable
+- List all the extractors with the ability to enable/disable the extractors
+- Filter listeners based on their support for file or dataset
+- Helm chart updated to support custom existing secret
+
+### Fixed
+
+- Clowder registration link on the top bar
+- Case-insensitive search
+- Download count immediately increments after download
+
 ## [v2.0.0-beta.2] - 2024-02-16
 
 ### Added

backend/Dockerfile

@@ -22,4 +22,5 @@ ENV PATH="/code/.venv/bin:$PATH"
 COPY ./app /code/app
 
 # launch app using uvicorn
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"]
+# Number of recommended workers is 2 x number_of_cores +1
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80", "--workers", "17"]

backend/app/config.py

@@ -9,7 +9,7 @@ class Settings(BaseSettings):
     API_V2_STR: str = "/api/v2"
     admin_email: str = "devnull@ncsa.illinois.edu"
     frontend_url: str = "http://localhost:3000"
-    version: str = "2.0.0-beta.2"
+    version: str = "2.0.0-beta.3"
 
     # Unique secret for hashing API keys. Generate with `openssl rand -hex 32`
     local_auth_secret = "clowder_secret_key"

backend/app/keycloak_auth.py

@@ -2,6 +2,7 @@
 import json
 import logging
 from datetime import datetime
+from typing import Optional
 
 from fastapi import Depends, HTTPException, Security
 from fastapi.security import APIKeyCookie, APIKeyHeader, OAuth2AuthorizationCodeBearer
@@ -435,6 +436,43 @@ async def create_user(email: str, password: str, firstName: str, lastName: str):
     return user
 
 
+async def update_user(
+    email: str,
+    new_email: Optional[str],
+    new_password: Optional[str],
+    new_firstName: Optional[str],
+    new_lastName: Optional[str],
+):
+    """Update existing user in Keycloak."""
+    keycloak_admin = KeycloakAdmin(
+        server_url=settings.auth_server_url,
+        username=settings.keycloak_username,
+        password=settings.keycloak_password,
+        realm_name=settings.keycloak_realm_name,
+        user_realm_name=settings.keycloak_user_realm_name,
+        # client_secret_key=settings.auth_client_secret,
+        # client_id=settings.keycloak_client_id,
+        verify=True,
+    )
+    existing_user_id = keycloak_admin.get_user_id(email)
+    existing_user = keycloak_admin.get_user(existing_user_id)
+    # Update user and set password
+    keycloak_admin.update_user(
+        existing_user_id,
+        {
+            "email": new_email or existing_user["email"],
+            "username": new_email or existing_user["email"],
+            "firstName": new_firstName or existing_user["firstName"],
+            "lastName": new_lastName or existing_user["lastName"],
+        },
+    )
+    if new_password:
+        keycloak_admin.set_user_password(existing_user_id, new_password, False)
+
+    updated_user = keycloak_admin.get_user(existing_user_id)
+    return updated_user
+
+
 def delete_user(email: str):
     """Create a user in Keycloak."""
     keycloak_admin = KeycloakAdmin(

backend/app/main.py

@@ -86,7 +86,7 @@
     description="A cloud native data management framework to support any research domain. Clowder was "
     "developed to help researchers and scientists in data intensive domains manage raw data, complex "
     "metadata, and automatic data pipelines. ",
-    version="2.0.0-beta.2",
+    version="2.0.0-beta.3",
     contact={"name": "Clowder", "url": "https://clowderframework.org/"},
     license_info={
         "name": "Apache 2.0",
@@ -316,7 +316,9 @@ async def startup_beanie():
             ThumbnailDBViewList,
             LicenseDB,
         ],
-        recreate_views=True,
+        # If view exists, will not recreate
+        # When view query changes, make sure to manually drop view and recreate
+        recreate_views=False,
     )
 
 

backend/app/models/metadata.py

@@ -307,9 +307,11 @@ async def validate_context(
             detail="Context is required",
         )
     if context is not None:
-        pass
+        # TODO validate context
+        return content
     if context_url is not None:
-        pass
+        # TODO validate context
+        return content
     if definition is not None:
         if (
             md_def := await MetadataDefinitionDB.find_one(
@@ -322,7 +324,7 @@ async def validate_context(
                 status_code=400,
                 detail=f"{definition} is not valid metadata definition",
             )
-    return content
+        return content
 
 
 def deep_update(orig: dict, new: dict):

backend/app/models/users.py

@@ -18,6 +18,12 @@ class UserIn(UserBase):
     password: str
 
 
+class UserUpdate(BaseModel):
+    first_name: Optional[str]
+    last_name: Optional[str]
+    password: Optional[str]
+
+
 class UserLogin(BaseModel):
     email: EmailStr
     password: str

backend/app/routers/authentication.py

@@ -5,15 +5,18 @@
     enable_disable_user,
     get_current_user,
     keycloak_openid,
+    update_user,
 )
 from app.models.datasets import DatasetDBViewList
-from app.models.users import UserDB, UserIn, UserLogin, UserOut
+from app.models.users import UserDB, UserIn, UserLogin, UserOut, UserUpdate
+from app.routers.utils import save_refresh_token
 from beanie import PydanticObjectId
 from fastapi import APIRouter, Depends, HTTPException
 from keycloak.exceptions import (
     KeycloakAuthenticationError,
     KeycloakGetError,
     KeycloakPostError,
+    KeycloakPutError,
 )
 from passlib.hash import bcrypt
 
@@ -69,6 +72,7 @@ async def save_user(userIn: UserIn):
 async def login(userIn: UserLogin):
     try:
         token = keycloak_openid.token(userIn.email, userIn.password)
+        await save_refresh_token(token["refresh_token"], userIn.email)
         return {"token": token["access_token"]}
     # bad credentials
     except KeycloakAuthenticationError as e:
@@ -95,6 +99,45 @@ async def authenticate_user(email: str, password: str):
     return user
 
 
+@router.patch("/users/me", response_model=UserOut)
+async def update_current_user(
+    userUpdate: UserUpdate, current_user=Depends(get_current_user)
+):
+    try:
+        await update_user(
+            current_user.email,
+            None,
+            userUpdate.password,
+            userUpdate.first_name,
+            userUpdate.last_name,
+        )
+    except KeycloakGetError as e:
+        raise HTTPException(
+            status_code=e.response_code,
+            detail=json.loads(e.error_message),
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    except KeycloakPutError as e:
+        raise HTTPException(
+            status_code=e.response_code,
+            detail=json.loads(e.error_message),
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Update local user
+    user = await UserDB.find_one(UserDB.email == current_user.email)
+
+    if userUpdate.first_name:
+        user.first_name = userUpdate.first_name
+    if userUpdate.last_name:
+        user.last_name = userUpdate.last_name
+    if userUpdate.password:
+        user.hashed_password = bcrypt.hash(userUpdate.password)
+
+    await user.save()
+    return user.dict()
+
+
 @router.get("/users/me/is_admin", response_model=bool)
 async def get_admin(
     dataset_id: str = None, current_username=Depends(get_current_user)