From 4672b7d412c64cbee6fa08a8f17e673c7333e6ef Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Sun, 11 Apr 2021 11:45:34 -0400 Subject: [PATCH 01/29] Add some stuff --- turtle_shell/README.rst | 230 +++++++++++++++++++++++++++++++ turtle_shell/function_to_form.py | 121 ++++++++++++++++ turtle_shell/models.py | 22 ++- turtle_shell/views.py | 30 +++- 4 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 turtle_shell/README.rst create mode 100644 turtle_shell/function_to_form.py diff --git a/turtle_shell/README.rst b/turtle_shell/README.rst new file mode 100644 index 0000000..256a614 --- /dev/null +++ b/turtle_shell/README.rst @@ -0,0 +1,230 @@ +Function to Form View +===================== + +Motivation +---------- + +1. You have a bunch of shell scripts lying around to do things. +2. You don't want to force everyone to install your dependencies or use docker or whatnot. +3. Your permission model isn't SOOO complicated that it's necessary to have everyone use their own tokens OR you're just doing read-only things. +4. You want people to have website goodness (deep-linking, record of actions, easy on-boarding, etc) +5. Composing and/or ability to long-poll endpoints seems cool to you. + +How does it work? +----------------- + + +This lil' old library converts _your_ function with annotations into a ✨Django Form✨ (that calls the function on ``form_valid``!) + +It leverages some neat features of defopt under the hood so that a function like this: + +.. code-block:: python + + def some_action( + username: str, + url: str, + optional_comment: str=None, + hits: int = 5, + _special_hidden_field: bool=False, + ): + """Perform action on URL for username. + + Args: + username: the user to associate the URL with + url: which url to hit + optional_comment: why this happened + hits: how many hits you saw + """ + pass + + +Becomes this awesome form! + + + + +Overall gist +------------ + +You register your functions with the library:: + + MyFuncWrapper = wrapit.wrap(myfunc) + +Then in urls.py:: + + + path("/myfunc", MyFuncWrapper.as_view()) + path("/api/myfunc", MyFuncWrapper.as_view(graphql=True)) + +And finally run migrations:: + + ... + + +Now you can get list view / form to create / graphql API to create. + +Example Implementation +---------------------- + +executions.py:: + + from easy_execute import ExecutionWrapper + from my_util_scripts import find_root_cause, summarize_issue, error_summary + + Registry = ExecutionWrapper() + + + FindRootCause = Registry.wrap(find_root_cause) + SummarizeIssue = Registry.wrap(summarize_issue) + ErrorSummary = Registry.wrap(error_summary) + + + + +You can just stop there if ya like! Woo :) + +For convenience, easy_execute provides a router that set ups default list/detail/edit by function. + +urls.py:: + + from executions import Registry + from graphene_django import GraphQLView + + router = Registry.get_router(list_template="list.html", detail_template="detail.html") + + urlpatterns = [ + path('/api', GraphQLView(schema=Registry.schema, include_graphiql=False)), + path('/graphql', GraphQLView(schema=Registry.schema, include_graphiql=True)), + # get default list and view together + path('/execute', include(router.urls), + ] + + # /execute/overview + # /execute/find-root-cause + # /execute/find-root-cause/create + # /execute/find-root-cause/ + # /execute/summarize-issue + # /execute/summarize-issue/create + # /execute/summarize-issue/ + +Of course you can also customize further:: + +views:: + + from . import executions + + class FindRootCauseList(executions.FindRootCause.list_view()): + template_name = "list-root-cause.html" + + class FindRootCauseDetail(executions.FindRootCause.detail_view()): + template_name = "detail-root-cause.html" + +These use the generic django views under the hood. + +What's missing from this idea +----------------------------- + +- granular permissions (gotta think about nice API for this) +- separate tables for different objects. + +Using the library +----------------- + + +ExecutionResult: + DB attributes: + - pk (UUID) + - input_json + - output_json + - func_name # defaults to module.function_name but can be customized + + Properties: + get_formatted_response() -> JSON serializable object + + +ExecutionForm(func) + +ExecutionGraphQLView(func) + + +Every function gets a generic output:: + + mutation { dxFindRootCause(input: {job_id: ..., project: ...}) { + uuid: str + execution { + status: String? + exitCode: Int + successful: Bool + } + rawOutput { + stderr: String? + stdout: String # often JSON serializable + } + } + errors: Optional { + type + message + } + } + + +But can also have structured output:: + + mutation { dxFindRootCause(input: {job_id: ..., project: ...}) { + output { + rootCause: ... + rootCauseMessage: ... + rootCauseLog: ... + } + } + } + +Other potential examples:: + + mutation { summarizeAnalysis(input: {analysisId: ...}) { + output { + fastqSizes { + name + size + } + undeterminedReads { + name + size + } + humanSummary + } + } + + +Which would look like (JSON as YAML):: + + output: + fastqSizes: + - name: "s_1.fastq.gz" + size: "125MB" + - name: "s_2.fastq.gz" + size: "125GB" + undeterminedReads: + humanSummary: "Distribution heavily skewed. 10 barcodes missing. 5 barcodes much higher than rest." + + + + +Why is this useful? +------------------- + +I had a bunch of defopt-based CLI tools that I wanted to expose as webapps for folks +who were not as command line savvy. + +1. Python type signatures are quite succinct - reduces form boilerplate +2. Expose utility functions as forms for users + + +Customizing the forms +--------------------- + +First - you can pass a config dictionary to ``function_to_form`` to tell it to +use particular widgets for fields or how to construct a form field for your custom type ( +as a callable that takes standard field keyword arguments). + +You can also subclass the generated form object to add your own ``clean_*`` methods or more complex validation - yay! diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py new file mode 100644 index 0000000..9b06db8 --- /dev/null +++ b/turtle_shell/function_to_form.py @@ -0,0 +1,121 @@ +""" +Function to a Form +------------------ + +Converts function signatures into django forms. + +NOTE: with enums it's recommended to use the string version, since the value will be used as the +representation to the user (and generally numbers aren't that valuable) +""" +import enum +import inspect +import re + +from django import forms +from defopt import Parameter, signature, _parse_docstring +from typing import Dict +from typing import Type + + +type2field_type = {int: forms.IntegerField, str: forms.CharField, bool: forms.BooleanField} + + +def doc_mapping(str) -> Dict[str, str]: + return {} + + +def function_to_form(func, config: dict = None) -> Type[forms.Form]: + """Convert a function to a Django Form. + + Args: + func: the function to be changed + config: A dictionary with keys ``widgets`` and ``fields`` each mapping types/specific + arguments to custom fields + """ + sig = signature(func) + # i.e., class body for form + fields = {} + for parameter in sig.parameters.values(): + fields[parameter.name] = param_to_field(parameter, config) + fields["__doc__"] = re.sub("\n+", "\n", _parse_docstring(inspect.getdoc(func)).text) + fields["_func"] = func + form_name = "".join(part.capitalize() for part in func.__name__.split("_")) + + def execute_function(self): + # TODO: reconvert back to enum type! :( + return func(**self.cleaned_data) + + fields["execute_function"] = execute_function + + return type(form_name, (forms.Form,), fields) + + +def param_to_field(param: Parameter, config: dict = None) -> forms.Field: + """Convert a specific arg to a django form field. + + See function_to_form for config definition.""" + config = config or {} + all_types = dict(type2field_type) + all_types.update(config.get("types", {})) + widgets = config.get("widgets") or {} + field_type = None + kwargs = {} + if issubclass(param.annotation, enum.Enum): + field_type = forms.TypedChoiceField + kwargs["coerce"] = param.annotation + kwargs["choices"] = [(member.value, member.value) for member in param.annotation] + # coerce back + if isinstance(param.default, param.annotation): + kwargs["initial"] = param.default.value + else: + for k, v in all_types.items(): + if isinstance(k, str): + if param.name == k: + field_type = v + break + continue + if issubclass(k, param.annotation): + field_type = v + break + else: + raise ValueError(f"Field {param.name}: Unknown field type: {param.annotation}") + if param.default is Parameter.empty: + kwargs["required"] = True + elif param.default is None: + kwargs["required"] = False + else: + kwargs["required"] = False + kwargs.setdefault("initial", param.default) + if param.doc: + kwargs["help_text"] = param.doc + for k, v in widgets.items(): + if isinstance(k, str): + if param.name == k: + kwargs["widget"] = v + break + continue + if issubclass(k, param.annotation): + kwargs["widget"] = v + break + return field_type(**kwargs) + + +def to_graphene_form_mutation(func): + import graphene + from graphene_django.forms.mutation import DjangoFormMutation + + form_klass = function_to_form(func) + + class DefaultOperationMutation(DjangoFormMutation): + form_output_json = graphene.String() + class Meta: + form_class = form_klass + + @classmethod + def perform_mutate(cls, form, info): + return cls(errors=[], from_output_json=json.dumps(form.execute_func())) + + + DefaultOperationMutation.__doc__ = f'Mutation form for {form_klass.__name__}.\n{form_klass.__doc__}' + DefaultOperationMutation.__name__ = f'{form_klass.__name__}Mutation' + return DefaultOperationMutation diff --git a/turtle_shell/models.py b/turtle_shell/models.py index 71a8362..a54a7bc 100644 --- a/turtle_shell/models.py +++ b/turtle_shell/models.py @@ -1,3 +1,23 @@ from django.db import models +import uuid + + + +class ExecutionResult(models.Model): + uuid = models.UUIDField(unique=True, editable=False, default=uuid.uuid4) + func_name = models.CharField(max_length=512) + input_json = models.JSONField() + output_json = models.JSONField() + + class ExecutionStatus(models.TextChoices): + CREATED = 'CREATED', 'Created' + RUNNING = 'RUNNING', 'Running' + DONE = 'DONE', 'Done' + ERRORED = 'ERRORED', 'Errored' + + status = models.CharField(max_length=10, choices=ExecutionStatus.choices, + default=ExecutionStatus.CREATED) + + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) -# Create your models here. diff --git a/turtle_shell/views.py b/turtle_shell/views.py index 91ea44a..cb764b1 100644 --- a/turtle_shell/views.py +++ b/turtle_shell/views.py @@ -1,3 +1,29 @@ -from django.shortcuts import render +from django.views.generic import DetailView +from django.views.generic import ListView +from django.views.generic.edit import CreateView +from .models import ExecutionResult -# Create your views here. + +class ExecutionViewMixin: + """Wrapper that auto-filters queryset to include function name""" + func_name: str = None + model = ExecutionResult + + def __init__(self, *a, **k): + super().__init__(*a, **k) + if not self.func_name: + raise ValueError("Must specify function name for ExecutionClasses classes (class was {type(self).__name__})") + + def get_queryset(self): + qs = super().get_queryset() + return qs.filter(func_name=self.func_name) + + +class ExecutionDetailView(DetailView, ExecutionViewMixin): + pass + +class ExecutionListView(ListView, ExecutionViewMixin): + pass + +class ExecutionCreateView(CreateView, ExecutionViewMixin): + pass From 6bebee554b6ca6d5f0927f032068ff17ea04f9ec Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Wed, 14 Apr 2021 22:33:09 -0400 Subject: [PATCH 02/29] License authors and .gitignore --- .gitignore | 27 +++++++++++++++++++++++++++ AUTHORS | 1 + LICENSE | 21 +++++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 .gitignore create mode 100644 AUTHORS create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5cb9b06 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +*.py[co] +.project +.pydevproject +*~ +*.db +*.orig +*.DS_Store +.tox +*.egg/* +.eggs/ +*.egg-info/* +build/* +docs/_build/* +dist/* + +*.log +nosetests*.xml + +.coverage +coverage.xml +coverage + +test.db +.venv + +*.sublime-workspace +.python-version diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..476348f --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Jeffrey Tratner @jtratner diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8d2b5a9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Jeff Tratner + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From eef531e6cf99a36a86f43e628ee0ddead81cd1ec Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Sun, 11 Apr 2021 11:46:08 -0400 Subject: [PATCH 03/29] Set up turtle shell and examples Mostly working model wooo Tests passing --- turtle_shell/README.rst | 25 +- turtle_shell/__init__.py | 70 +++++ turtle_shell/function_to_form.py | 117 +++++--- turtle_shell/migrations/0001_initial.py | 27 ++ turtle_shell/models.py | 69 ++++- .../turtle_shell/executionresult_detail.html | 25 ++ .../turtle_shell/executionresult_list.html | 13 + .../templates/turtle_shell/overview.html | 11 + turtle_shell/tests.py | 3 - turtle_shell/tests/__init__.py | 0 turtle_shell/tests/conftest.py | 4 + turtle_shell/tests/test_django_cli2ui.py | 250 ++++++++++++++++++ turtle_shell/tests/test_form_comparisons.py | 129 +++++++++ turtle_shell/tests/utils.py | 33 +++ turtle_shell/views.py | 61 ++++- 15 files changed, 785 insertions(+), 52 deletions(-) create mode 100644 turtle_shell/migrations/0001_initial.py create mode 100644 turtle_shell/templates/turtle_shell/executionresult_detail.html create mode 100644 turtle_shell/templates/turtle_shell/executionresult_list.html create mode 100644 turtle_shell/templates/turtle_shell/overview.html delete mode 100644 turtle_shell/tests.py create mode 100644 turtle_shell/tests/__init__.py create mode 100644 turtle_shell/tests/conftest.py create mode 100644 turtle_shell/tests/test_django_cli2ui.py create mode 100644 turtle_shell/tests/test_form_comparisons.py create mode 100644 turtle_shell/tests/utils.py diff --git a/turtle_shell/README.rst b/turtle_shell/README.rst index 256a614..f3f5e33 100644 --- a/turtle_shell/README.rst +++ b/turtle_shell/README.rst @@ -1,6 +1,16 @@ Function to Form View ===================== +CURRENTLY BROKEN STUFF: + +1. Graphql view does not respect defaults :( (e.g., find_root_cause comes through as false :() +2. Needs testing +3. Needs lots of testing + +LIMITATIONS: + +1. Boolean fields always are passed in as False. + Motivation ---------- @@ -48,13 +58,22 @@ Overall gist You register your functions with the library:: - MyFuncWrapper = wrapit.wrap(myfunc) + Registry = turtle_shell.get_registry() + + Registry.add(myfunc) Then in urls.py:: - path("/myfunc", MyFuncWrapper.as_view()) - path("/api/myfunc", MyFuncWrapper.as_view(graphql=True)) + import turtle_shell + + path("execute/", include(turtle_shell.get_registry().get_router().urls)l) + +If you want GraphQL, then [install graphene-django](https://docs.graphene-python.org/projects/django/en/latest/installation/) +and put into installed apps (also django filter), then finally:: + + path("api", GraphQLView.as_view(schema=turtle_shell.get_registry().schema, graphiql=False)), + path("graphql", GraphQLView.as_view(schema=turtle_shell.get_registry().schema, graphiql=True)) And finally run migrations:: diff --git a/turtle_shell/__init__.py b/turtle_shell/__init__.py index e69de29..1294d72 100644 --- a/turtle_shell/__init__.py +++ b/turtle_shell/__init__.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass + +@dataclass +class _Router: + urls: list + +class _Registry: + func_name2func: dict + _schema = None + + def __init__(self): + self.func_name2func = {} + + @classmethod + def get_registry(self): + return _RegistrySingleton + + def add(self, func, name=None): + from .function_to_form import _Function + + # TODO: maybe _Function object should have overridden __new__ to keep it immutable?? :-/ + name = name or func.__name__ + func_obj = self.get(name) + if not func_obj: + func_obj = _Function.from_function(func, name=name) + self.func_name2func[func_obj.name] = func_obj + else: + if func_obj.func is not func: + raise ValueError(f"Func {name} already registered. (existing is {func_obj})") + return self.get(name) + + def get(self, name): + return self.func_name2func.get(name, None) + + def summary_view(self, request): + from django.template import loader + from django.http import HttpResponse + + template = loader.get_template("turtle_shell/overview.html") + context = {"registry": self, "functions": self.func_name2func.values()} + return HttpResponse(template.render(context)) + + def get_router(self, *, list_template= 'turtle_shell/executionresult_list.html', + detail_template= 'turtle_shell/executionresult_detail.html', create_template= + 'turtle_shell/executionresult_create.html'): + from django.urls import path + from . import views + urls = [path("", self.summary_view, name="overview")] + for func in self.func_name2func.values(): + urls.extend(views.Views.from_function(func).urls(list_template=list_template, + detail_template=detail_template, create_template=create_template)) + return _Router(urls=(urls, "turtle_shell")) + + def clear(self): + self.func_name2func.clear() + self._schema = None + assert not self.func_name2func + + @property + def schema(self): + from .graphene_adapter import schema_for_registry + + if not self._schema: + self._schema = schema_for_registry(self) + + return self._schema + + +_RegistrySingleton = _Registry() +get_registry = _Registry.get_registry diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index 9b06db8..99832a8 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -10,21 +10,37 @@ import enum import inspect import re +import typing +from dataclasses import dataclass from django import forms from defopt import Parameter, signature, _parse_docstring -from typing import Dict +from typing import Dict, Optional from typing import Type +import pathlib -type2field_type = {int: forms.IntegerField, str: forms.CharField, bool: forms.BooleanField} +type2field_type = {int: forms.IntegerField, str: forms.CharField, bool: forms.BooleanField, + Optional[bool]: forms.NullBooleanField, + pathlib.Path: forms.CharField, dict: forms.JSONField} + +@dataclass +class _Function: + func: callable + name: str + form_class: object + + @classmethod + def from_function(cls, func, *, name): + from turtle_shell.function_to_form import function_to_form + return cls(func=func, name=name, form_class=function_to_form(func, name=name)) def doc_mapping(str) -> Dict[str, str]: return {} -def function_to_form(func, config: dict = None) -> Type[forms.Form]: +def function_to_form(func, *, config: dict = None, name: str=None) -> Type[forms.Form]: """Convert a function to a Django Form. Args: @@ -32,22 +48,59 @@ def function_to_form(func, config: dict = None) -> Type[forms.Form]: config: A dictionary with keys ``widgets`` and ``fields`` each mapping types/specific arguments to custom fields """ + name = name or func.__qualname__ sig = signature(func) # i.e., class body for form fields = {} + defaults = {} for parameter in sig.parameters.values(): fields[parameter.name] = param_to_field(parameter, config) + if parameter.default is not Parameter.empty: + defaults[parameter.name] = parameter.default fields["__doc__"] = re.sub("\n+", "\n", _parse_docstring(inspect.getdoc(func)).text) - fields["_func"] = func form_name = "".join(part.capitalize() for part in func.__name__.split("_")) - def execute_function(self): - # TODO: reconvert back to enum type! :( - return func(**self.cleaned_data) - fields["execute_function"] = execute_function + class BaseForm(forms.Form): + _func = func + _input_defaults = defaults + # use this for ignoring extra args from createview and such + def __init__(self, *a, instance=None, user=None, **k): + from crispy_forms.helper import FormHelper + from crispy_forms.layout import Submit + + super().__init__(*a, **k) + self.user = user + self.helper = FormHelper(self) + self.helper.add_input(Submit('submit', 'Execute!')) + + def execute_function(self): + # TODO: reconvert back to enum type! :( + return func(**self.cleaned_data) + + def save(self): + from .models import ExecutionResult + obj = ExecutionResult(func_name=name, + input_json=self.cleaned_data, + user=self.user) + obj.save() + return obj + + + return type(form_name, (BaseForm,), fields) + - return type(form_name, (forms.Form,), fields) +def is_optional(annotation): + if args := typing.get_args(annotation): + return len(args) == 2 and args[-1] == type(None) + + +def get_type_from_annotation(param: Parameter): + if is_optional(param.annotation): + return typing.get_args(param.annotation)[0] + if typing.get_origin(param.annotation): + raise ValueError(f"Field {param.name}: type class {param.annotation} not supported") + return param.annotation def param_to_field(param: Parameter, config: dict = None) -> forms.Field: @@ -60,24 +113,29 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: widgets = config.get("widgets") or {} field_type = None kwargs = {} - if issubclass(param.annotation, enum.Enum): + kind = get_type_from_annotation(param) + is_enum_class = False + try: + is_enum_class = issubclass(kind, enum.Enum) + except TypeError: + # e.g. stupid generic type stuff + pass + if is_enum_class: field_type = forms.TypedChoiceField - kwargs["coerce"] = param.annotation - kwargs["choices"] = [(member.value, member.value) for member in param.annotation] + kwargs["coerce"] = kind + kwargs["choices"] = [(member.value, member.value) for member in kind] # coerce back - if isinstance(param.default, param.annotation): + if isinstance(param.default, kind): kwargs["initial"] = param.default.value else: + field_type = all_types.get(param.annotation, all_types.get(kind)) for k, v in all_types.items(): - if isinstance(k, str): - if param.name == k: - field_type = v - break - continue - if issubclass(k, param.annotation): + if field_type: + break + if inspect.isclass(k) and issubclass(kind, k) or k == kind: field_type = v break - else: + if not field_type: raise ValueError(f"Field {param.name}: Unknown field type: {param.annotation}") if param.default is Parameter.empty: kwargs["required"] = True @@ -100,22 +158,3 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: return field_type(**kwargs) -def to_graphene_form_mutation(func): - import graphene - from graphene_django.forms.mutation import DjangoFormMutation - - form_klass = function_to_form(func) - - class DefaultOperationMutation(DjangoFormMutation): - form_output_json = graphene.String() - class Meta: - form_class = form_klass - - @classmethod - def perform_mutate(cls, form, info): - return cls(errors=[], from_output_json=json.dumps(form.execute_func())) - - - DefaultOperationMutation.__doc__ = f'Mutation form for {form_klass.__name__}.\n{form_klass.__doc__}' - DefaultOperationMutation.__name__ = f'{form_klass.__name__}Mutation' - return DefaultOperationMutation diff --git a/turtle_shell/migrations/0001_initial.py b/turtle_shell/migrations/0001_initial.py new file mode 100644 index 0000000..ba7b2c0 --- /dev/null +++ b/turtle_shell/migrations/0001_initial.py @@ -0,0 +1,27 @@ +# Generated by Django 3.1.8 on 2021-04-11 16:41 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ExecutionResult', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True)), + ('func_name', models.CharField(max_length=512)), + ('input_json', models.JSONField()), + ('output_json', models.JSONField()), + ('status', models.CharField(choices=[('CREATED', 'Created'), ('RUNNING', 'Running'), ('DONE', 'Done'), ('ERRORED', 'Errored')], default='CREATED', max_length=10)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + ] diff --git a/turtle_shell/models.py b/turtle_shell/models.py index a54a7bc..dbc2a29 100644 --- a/turtle_shell/models.py +++ b/turtle_shell/models.py @@ -1,23 +1,86 @@ from django.db import models +from django.urls import reverse +from django.conf import settings import uuid +class CaughtException(Exception): + """An exception that was caught and saved. Generally don't need to rollback transaction with + this one :)""" + def __init__(self, exc, message): + self.exc = exc + super().__init__(message) + + +class ResultJSONEncodeException(CaughtException): + """Exceptions for when we cannot save result as actual JSON field :(""" + class ExecutionResult(models.Model): - uuid = models.UUIDField(unique=True, editable=False, default=uuid.uuid4) - func_name = models.CharField(max_length=512) + uuid = models.UUIDField(primary_key=True, unique=True, editable=False, default=uuid.uuid4) + func_name = models.CharField(max_length=512, editable=False) input_json = models.JSONField() - output_json = models.JSONField() + output_json = models.JSONField(default=dict, null=True) + error_json = models.JSONField(default=dict, null=True) class ExecutionStatus(models.TextChoices): CREATED = 'CREATED', 'Created' RUNNING = 'RUNNING', 'Running' DONE = 'DONE', 'Done' ERRORED = 'ERRORED', 'Errored' + JSON_ERROR = 'JSON_ERROR', 'Result could not be coerced to JSON' status = models.CharField(max_length=10, choices=ExecutionStatus.choices, default=ExecutionStatus.CREATED) created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT, null=True) + + def execute(self): + """Execute with given input, returning caught exceptions as necessary""" + if self.status not in (self.ExecutionStatus.CREATED, self.ExecutionStatus.RUNNING): + raise ValueError("Cannot run - execution state isn't complete") + func = self.get_function() + try: + # TODO: redo conversion another time! + result = func(**self.input_json) + except Exception as e: + # TODO: catch integrity error separately + self.error_json = {"type": type(e).__name__, "message": str(e)} + self.status = self.ExecutionStatus.ERRORED + self.save() + raise CaughtException(f"Failed on {self.func_name} ({type(e).__name__})", e) from e + try: + self.output_json = result + self.status = self.ExecutionStatus.DONE + self.save() + except TypeError as e: + self.error_json = {"type": type(e).__name__, "message": str(e)} + msg = f"Failed on {self.func_name} ({type(e).__name__})" + if 'JSON serializable' in str(e): + self.status = self.ExecutionStatus.JSON_ERROR + # save it as a str so we can at least have something to show + self.output_json = str(result) + self.save() + raise ResultJSONEncodeException(msg, e) from e + else: + raise e + return self + + def get_function(self): + # TODO: figure this out + from . import get_registry + + func_obj = get_registry().get(self.func_name) + if not func_obj: + raise ValueError(f"No registered function defined for {self.func_name}") + return func_obj.func + + def get_absolute_url(self): + # TODO: prob better way to do this so that it all redirects right :( + return reverse(f'turtle_shell:detail-{self.func_name}', kwargs={"pk": self.pk}) + def __repr__(self): + return (f'<{type(self).__name__}(pk="{self.pk}", func_name="{self.func_name}",' + f' created={self.created}, modified={self.modified})') diff --git a/turtle_shell/templates/turtle_shell/executionresult_detail.html b/turtle_shell/templates/turtle_shell/executionresult_detail.html new file mode 100644 index 0000000..2808ccc --- /dev/null +++ b/turtle_shell/templates/turtle_shell/executionresult_detail.html @@ -0,0 +1,25 @@ +{% extends 'base.html' %} + +{% block content %} +

Execution for {{func_name}} ({{object.pk}})

+

State

+

{{object.status}}

+

Input

+
{{object.input_json|pprint|escape}}
+{% if object.output_json %} +

Output

+
{{object.output_json|pprint|escape}}
+{% endif %} +{% if object.error_json %} +

Error

+
{{object.error_json|pprint|escape}}
+{% endif %} +

User

+{{object.user}} +

Created

+{{object.created}} +

Modified

+{{object.modified}} +{% endblock content %} + + diff --git a/turtle_shell/templates/turtle_shell/executionresult_list.html b/turtle_shell/templates/turtle_shell/executionresult_list.html new file mode 100644 index 0000000..e742e19 --- /dev/null +++ b/turtle_shell/templates/turtle_shell/executionresult_list.html @@ -0,0 +1,13 @@ +{% extends 'base.html' %} + +{% block content %} +

Executions for {{func_name}}

+

+ +{% endblock content %} + + diff --git a/turtle_shell/templates/turtle_shell/overview.html b/turtle_shell/templates/turtle_shell/overview.html new file mode 100644 index 0000000..452577f --- /dev/null +++ b/turtle_shell/templates/turtle_shell/overview.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} + +{% block content %} + +{% endblock content %} diff --git a/turtle_shell/tests.py b/turtle_shell/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/turtle_shell/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/turtle_shell/tests/__init__.py b/turtle_shell/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/turtle_shell/tests/conftest.py b/turtle_shell/tests/conftest.py new file mode 100644 index 0000000..88e73b8 --- /dev/null +++ b/turtle_shell/tests/conftest.py @@ -0,0 +1,4 @@ +import pytest + +# ensure we get pretty pytest-style diffs in this module :) +pytest.register_assert_rewrite("tests.utils") diff --git a/turtle_shell/tests/test_django_cli2ui.py b/turtle_shell/tests/test_django_cli2ui.py new file mode 100644 index 0000000..f03b6ac --- /dev/null +++ b/turtle_shell/tests/test_django_cli2ui.py @@ -0,0 +1,250 @@ +import enum +import json + +import pytest +from django import forms + +from .utils import compare_form_field +from turtle_shell.function_to_form import param_to_field +import turtle_shell +from defopt import Parameter +from typing import Optional +import typing + + +class Color(enum.Enum): + red = enum.auto() + green = enum.auto() + yellow = enum.auto() + + +COLOR_CHOICES = [(e.value, e.value) for e in Color] + + +class Flag(enum.Enum): + is_apple = "is_apple" + is_banana = "is_banana" + + +FLAG_CHOICES = [(e.value, e.value) for e in Flag] + + +class Text(str): + pass + + +def example_func( + *, + int_arg: int, + int_arg_with_default: int = 5, + bool_arg_default_true: bool = True, + bool_arg_default_false: bool = False, + bool_arg_no_default: bool, + str_arg: str, + str_arg_with_default: str = "whatever", + text_arg: Text, + text_arg_with_default: Text = Text("something"), + enum_auto: Color, + enum_auto_not_required: Color = None, + enum_auto_with_default: Color = Color.green, + enum_str: Flag, + enum_str_with_default: Flag = Flag.is_apple, + undocumented_arg: str = None, +): + """ + First line of text content should be there. + + Args: + int_arg: Browser native int field + int_arg_with_default: Browser native int field with a default + bool_arg_default_true: should be checked checkbox + bool_arg_default_false: should be unchecked checkbox + bool_arg_no_default: Bool arg with a dropdown + str_arg: should be small field + str_arg_with_default: should be small field with text in it + text_arg: should have a big text field + text_arg_with_default: should have big text field with something filled in + enum_auto: should be choices with key names + enum_auto_not_required: choice field not required + enum_auto_with_default: choice field with entry selected + enum_str: should be choices with string values + + Later lines of content here. + + """ + + +class ExpectedFormForExampleFunc(forms.Form): + """ + First line of text content should be there. + + Later lines of content here. + """ + + int_arg = forms.IntegerField() + int_arg_with_default = forms.IntegerField(initial=5) + bool_arg_default_true = forms.BooleanField(initial=True) + bool_arg_default_false = forms.BooleanField(initial=False) + bool_arg_no_default = forms.BooleanField() + str_arg = forms.CharField() + str_arg_with_default = forms.CharField(initial="whatever") + # TODO: different widget + text_arg = forms.CharField() + # TODO: different widget + text_arg_with_default = forms.CharField(initial="something") + enum_auto = forms.ChoiceField + + +def _make_parameter(name, annotation, doc="", **kwargs): + """helper for simple params :) """ + return Parameter( + name=name, + kind=Parameter.KEYWORD_ONLY, + default=kwargs.get("default", Parameter.empty), + annotation=annotation, + doc=doc, + ) + + +@pytest.mark.parametrize( + "arg,expected", + [ + (_make_parameter("int", int, ""), forms.IntegerField(required=True)), + ( + _make_parameter("int_none_default_not_required", int, default=None), + forms.IntegerField(required=False), + ), + ( + _make_parameter("int_default", int, default=-1), + forms.IntegerField(initial=-1, required=False), + ), + ( + _make_parameter("str_doc", str, "some doc"), + forms.CharField(help_text="some doc", required=True), + ), + ( + _make_parameter("str_doc_not_required", str, "some doc", default="a"), + forms.CharField(required=False, initial="a", help_text="some doc"), + ), + ( + _make_parameter("str_doc_falsey", str, "some doc", default=""), + forms.CharField(initial="", required=False, help_text="some doc"), + ), + ( + _make_parameter("bool_falsey", bool, "some doc", default=False), + forms.BooleanField(required=False, initial=False, help_text="some doc"), + ), + ( + _make_parameter("bool_truthy", bool, "some doc", default=True), + forms.BooleanField(required=False, initial=True, help_text="some doc"), + ), + ( + _make_parameter("bool_required", bool, "some doc"), + forms.BooleanField(required=True, help_text="some doc"), + ), + ( + _make_parameter("enum_auto", Color), + forms.TypedChoiceField(coerce=Color, choices=COLOR_CHOICES), + ), + ( + _make_parameter("enum_auto_default", Color, "another doc", default=Color.green), + forms.TypedChoiceField( + coerce=Color, + initial=Color.green.value, + choices=COLOR_CHOICES, + required=False, + help_text="another doc", + ), + ), + ( + _make_parameter("enum_str", Flag), + forms.TypedChoiceField(coerce=Flag, choices=FLAG_CHOICES), + ), + ( + _make_parameter("enum_str_default", Flag, default=Flag.is_apple), + forms.TypedChoiceField( + coerce=Flag, initial="is_apple", choices=FLAG_CHOICES, required=False + ), + ), + ( + _make_parameter("optional_bool", Optional[bool], default=True), + forms.NullBooleanField(initial=True, required=False) + ), + ], + ids=lambda x: x.name if hasattr(x, "name") else x, +) +def test_convert_arg(arg, expected): + field = param_to_field(arg) + compare_form_field(arg.name, field, expected) + + +def test_custom_widgets(): + param = _make_parameter("str_large_text_field", Text, "some doc", default="") + text_input_widget = forms.TextInput(attrs={"size": "80", "autocomplete": "off"}) + compare_form_field( + "str_large_text_field", + param_to_field( + param, {"widgets": {Text: text_input_widget}, "types": {Text: forms.CharField}}, + ), + forms.CharField( + widget=text_input_widget, help_text="some doc", initial="", required=False + ), + ) + + +def test_validators(): + # something about fields failing validation + pass + + +def execute_gql_and_get_input_json(func, gql): + registry = turtle_shell.get_registry() + registry.clear() + registry.add(func) + result = registry.schema.execute(gql) + data = result.data + assert not result.errors + return json.loads(list(data.values())[0]["result"]["inputJson"]) + # data = json.loads(result["data"]["result"]["inputJson"]) + # return data + +def test_defaults(db): + # defaults should be passed through + def myfunc(a: bool=True, b: str="whatever"): + pass + + resp = execute_gql_and_get_input_json(myfunc, "mutation { executeMyfunc(input: {}) { result { inputJson }}}") + assert resp == {"a": True, "b": "whatever"} + +@pytest.mark.xfail +def test_default_none(db): + + # defaults should be passed through + def myfunc(a: bool=None, b: str=None): + pass + + resp = execute_gql_and_get_input_json(myfunc, "mutation { executeMyfunc(input: {}) { result { inputJson }}}") + # sadly None's get replaced :( + assert resp == {"a": None, "b": None} + +def test_error_with_no_default(db): + + # no default should error + def my_func(*, a: bool, b: str): + pass + registry = turtle_shell.get_registry() + registry.clear() + registry.add(my_func) + gql = "mutation { executeMyfunc(input: {}) { result { inputJson }}}" + result = registry.schema.execute(gql) + assert result.errors + + +@pytest.mark.parametrize( + "parameter,exception_type,msg_regex", + [ + (_make_parameter("union_type", typing.Union[bool, str]), ValueError, "type class.*not supported"), + ]) +def test_exceptions(parameter, exception_type, msg_regex): + with pytest.raises(exception_type, match=msg_regex): + param_to_field(parameter) diff --git a/turtle_shell/tests/test_form_comparisons.py b/turtle_shell/tests/test_form_comparisons.py new file mode 100644 index 0000000..96652f4 --- /dev/null +++ b/turtle_shell/tests/test_form_comparisons.py @@ -0,0 +1,129 @@ +""" +Confirm that our form comparison tool actually works! +""" +import enum +import pytest +from django import forms +from typing import Type +from .utils import compare_form_field, compare_forms + + +class FormA(forms.Form): + myfield = forms.CharField() + + +class DifferentFieldNames(forms.Form): + otherfield = forms.CharField() + + +class DifferentFieldTypes(forms.Form): + myfield = forms.IntegerField() + + +class DifferentFieldRequired(forms.Form): + myfield = forms.CharField(required=False) + + +class DifferentInitial(forms.Form): + myfield = forms.CharField(initial="something") + + +class DocForm(forms.Form): + """ + My Doc + + Some new line + """ + + some = forms.CharField() + + +class DocFormExtraWhitespace(forms.Form): + """ + My Doc + + Some new line + """ + + some = forms.CharField() + + +class DifferentDoc(forms.Form): + """ + Another doc + + Some new line + """ + + some = forms.CharField() + + +def test_comparison(): + + compare_forms(FormA, FormA) + with pytest.raises(AssertionError, match="Forms have different field names"): + test_different_fields(raiseit=True) + + with pytest.raises(AssertionError, match=r"Field mismatch \(myfield\)"): + test_field_required(raiseit=True) + + with pytest.raises(AssertionError, match=r"Field mismatch \(myfield\)"): + test_field_initial(raiseit=True) + + with pytest.raises(AssertionError, match=r"Field type mismatch \(myfield\)"): + test_different_field_types(raiseit=True) + + with pytest.raises(AssertionError, match=r"Forms have different docstrings"): + test_doc_diff(raiseit=True) + + test_different_whitespace() + + +def _wrap_knownfail(f): + def wrapper(*args, raiseit=False, **kwargs): + try: + f(*args, **kwargs) + except AssertionError as exc: + if raiseit: + raise + else: + print("Known fail: {type(exc)}: {exc}") + + return wrapper + + +@_wrap_knownfail +def test_different_fields(): + compare_forms(FormA, DifferentFieldNames) + + +@_wrap_knownfail +def test_different_field_types(): + compare_forms(FormA, DifferentFieldTypes) + + +@_wrap_knownfail +def test_field_required(): + compare_forms(FormA, DifferentFieldRequired) + + +@_wrap_knownfail +def test_field_initial(): + compare_forms(FormA, DifferentInitial) + + +@_wrap_knownfail +def test_convert_form(): + pass + + +@_wrap_knownfail +def test_doc_diff(): + actual_fields = DocForm.declared_fields + expected_fields = DifferentDoc.declared_fields + assert actual_fields.keys() == expected_fields.keys(), "Different fields" + compare_forms(DocForm, DifferentDoc) + + +def test_different_whitespace(): + compare_forms(DocForm, DocFormExtraWhitespace) diff --git a/turtle_shell/tests/utils.py b/turtle_shell/tests/utils.py new file mode 100644 index 0000000..1d9f47a --- /dev/null +++ b/turtle_shell/tests/utils.py @@ -0,0 +1,33 @@ +from typing import Type +import inspect + +from django import forms + + +def compare_form_field(name, actual, expected): + """Compare important variables of two form fields""" + assert type(actual) == type(expected), f"Field type mismatch ({name})" + actual_vars = vars(actual) + expected_vars = vars(expected) + actual_vars.pop("widget", None) + expected_vars.pop("widget", None) + assert actual_vars == expected_vars, f"Field mismatch ({name})" + + +def compare_forms(actual: Type[forms.Form], expected: Type[forms.Form]): + """Compare two forms. + + Checks: + 1. Shared fields are the same (see compare_form_field) + 2. Both forms have the same set of fields + 3. Both forms have the same docstring + """ + actual_fields = actual.declared_fields + expected_fields = expected.declared_fields + shared_keys = list(set(actual_fields.keys()) & set(expected_fields.keys())) + for name in shared_keys: + actual_field = actual_fields[name] + expected_field = expected_fields[name] + compare_form_field(name, actual_field, expected_field) + assert actual_fields.keys() == expected_fields.keys(), "Forms have different field names" + assert inspect.getdoc(actual) == inspect.getdoc(expected), "Forms have different docstrings" diff --git a/turtle_shell/views.py b/turtle_shell/views.py index cb764b1..34ca291 100644 --- a/turtle_shell/views.py +++ b/turtle_shell/views.py @@ -1,7 +1,11 @@ from django.views.generic import DetailView from django.views.generic import ListView +from django.views.generic import TemplateView from django.views.generic.edit import CreateView from .models import ExecutionResult +from dataclasses import dataclass +from django.urls import path +from django.contrib import messages class ExecutionViewMixin: @@ -18,12 +22,61 @@ def get_queryset(self): qs = super().get_queryset() return qs.filter(func_name=self.func_name) + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['func_name'] = self.func_name + return context -class ExecutionDetailView(DetailView, ExecutionViewMixin): - pass -class ExecutionListView(ListView, ExecutionViewMixin): +class ExecutionDetailView(ExecutionViewMixin, DetailView): pass -class ExecutionCreateView(CreateView, ExecutionViewMixin): +class ExecutionListView(ExecutionViewMixin, ListView): pass + +class ExecutionCreateView(ExecutionViewMixin, CreateView): + + def get_form_kwargs(self, *a, **k): + kwargs = super().get_form_kwargs(*a, **k) + kwargs['user'] = self.request.user + return kwargs + + def form_valid(self, form): + from .models import CaughtException + sup = super().form_valid(form) + try: + self.object.execute() + except CaughtException as e: + messages.warning(self.request, str(e)) + else: + messages.info(self.request, f"Completed execution for {self.object.pk}") + return sup + + +@dataclass +class Views: + detail_view: object + list_view: object + create_view: object + func_name: str + + @classmethod + def from_function(cls, func: 'turtle_shell._Function'): + detail_view = type(f'{func.name}DetailView', (ExecutionDetailView,), ({"func_name": func.name})) + list_view = type(f'{func.name}ListView', (ExecutionListView,), ({"func_name": func.name})) + create_view = type(f'{func.name}CreateView', (ExecutionCreateView,), ({"func_name": + func.name, "form_class": func.form_class})) + return cls(detail_view=detail_view, list_view=list_view, create_view=create_view, + func_name=func.name) + + def urls(self, *, list_template, detail_template, create_template): + # TODO: namespace this again! + return [ + path(f"{self.func_name}/", self.list_view.as_view(template_name=list_template), + name=f"list-{self.func_name}"), + path(f"{self.func_name}/create/", + self.create_view.as_view(template_name=create_template), name=f"create-{self.func_name}"), + path(f"{self.func_name}//", + self.detail_view.as_view(template_name=detail_template), name=f"detail-{self.func_name}") + ] + From e55e7405ab4986bea2f0ce66d9f0320bddd23d23 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Sun, 11 Apr 2021 21:25:47 -0400 Subject: [PATCH 04/29] Do better job with enums --- turtle_shell/function_to_form.py | 2 +- turtle_shell/tests/conftest.py | 2 +- turtle_shell/tests/test_django_cli2ui.py | 51 +++++++++++++-------- turtle_shell/tests/test_form_comparisons.py | 23 +++++++++- turtle_shell/tests/utils.py | 30 ++++++++++-- 5 files changed, 83 insertions(+), 25 deletions(-) diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index 99832a8..f1c773b 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -123,7 +123,7 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: if is_enum_class: field_type = forms.TypedChoiceField kwargs["coerce"] = kind - kwargs["choices"] = [(member.value, member.value) for member in kind] + kwargs["choices"] = [(member.name, member.value) for member in kind] # coerce back if isinstance(param.default, kind): kwargs["initial"] = param.default.value diff --git a/turtle_shell/tests/conftest.py b/turtle_shell/tests/conftest.py index 88e73b8..4cf7b3d 100644 --- a/turtle_shell/tests/conftest.py +++ b/turtle_shell/tests/conftest.py @@ -1,4 +1,4 @@ import pytest # ensure we get pretty pytest-style diffs in this module :) -pytest.register_assert_rewrite("tests.utils") +pytest.register_assert_rewrite("turtle_shell.tests.utils") diff --git a/turtle_shell/tests/test_django_cli2ui.py b/turtle_shell/tests/test_django_cli2ui.py index f03b6ac..bd1f7e4 100644 --- a/turtle_shell/tests/test_django_cli2ui.py +++ b/turtle_shell/tests/test_django_cli2ui.py @@ -4,8 +4,9 @@ import pytest from django import forms -from .utils import compare_form_field +from .utils import compare_form_field, compare_forms from turtle_shell.function_to_form import param_to_field +from turtle_shell.function_to_form import function_to_form import turtle_shell from defopt import Parameter from typing import Optional @@ -18,7 +19,7 @@ class Color(enum.Enum): yellow = enum.auto() -COLOR_CHOICES = [(e.value, e.value) for e in Color] +COLOR_CHOICES = [(e.name, e.value) for e in Color] class Flag(enum.Enum): @@ -75,24 +76,38 @@ def example_func( class ExpectedFormForExampleFunc(forms.Form): - """ - First line of text content should be there. - - Later lines of content here. - """ - - int_arg = forms.IntegerField() - int_arg_with_default = forms.IntegerField(initial=5) - bool_arg_default_true = forms.BooleanField(initial=True) - bool_arg_default_false = forms.BooleanField(initial=False) - bool_arg_no_default = forms.BooleanField() - str_arg = forms.CharField() - str_arg_with_default = forms.CharField(initial="whatever") + """\nFirst line of text content should be there. \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \nLater lines of content here.""" + # extra whitespace cuz of weird docstring parsing from defopt + int_arg = forms.IntegerField(help_text="Browser native int field", required=True) + int_arg_with_default = forms.IntegerField(initial=5, help_text="Browser native int field with a default", required=False) + bool_arg_default_true = forms.BooleanField(initial=True, help_text="should be checked checkbox", required=False) + bool_arg_default_false = forms.BooleanField(initial=False, help_text="should be unchecked checkbox", required=False) + bool_arg_no_default = forms.BooleanField(help_text="Bool arg with a dropdown", required=True) + str_arg = forms.CharField(help_text="should be small field", required=True) + str_arg_with_default = forms.CharField(initial="whatever", help_text="should be small field with text in it", required=False) # TODO: different widget - text_arg = forms.CharField() + text_arg = forms.CharField(help_text="should have a big text field", required=True) # TODO: different widget - text_arg_with_default = forms.CharField(initial="something") - enum_auto = forms.ChoiceField + text_arg_with_default = forms.CharField(initial="something", help_text="should have big text field with something filled in", required=False) + undocumented_arg = forms.CharField(required=False) + enum_auto = forms.TypedChoiceField(choices=COLOR_CHOICES, required=True, help_text="should be choices with key names", coerce=Color) + enum_auto_not_required = forms.TypedChoiceField(choices=COLOR_CHOICES, required=False, coerce=Color, + help_text="choice field not required") + enum_auto_with_default = forms.TypedChoiceField(choices=COLOR_CHOICES, + initial=Color.green.value, required=False, + coerce=Color, help_text="choice field with entry selected") + enum_str = forms.TypedChoiceField(choices=[("is_apple", "is_apple"), ("is_banana", + "is_banana")], required=True, coerce=Flag, help_text="should be choices with string values") + enum_str_with_default = forms.TypedChoiceField(choices=[("is_apple", "is_apple"), ("is_banana", + "is_banana")], required=False, initial=Flag.is_apple.value, + coerce=Flag) + + +def test_compare_complex_example(db): + actual = function_to_form(example_func) + compare_forms(actual, ExpectedFormForExampleFunc) + + def _make_parameter(name, annotation, doc="", **kwargs): diff --git a/turtle_shell/tests/test_form_comparisons.py b/turtle_shell/tests/test_form_comparisons.py index 96652f4..130e9c2 100644 --- a/turtle_shell/tests/test_form_comparisons.py +++ b/turtle_shell/tests/test_form_comparisons.py @@ -61,7 +61,7 @@ class DifferentDoc(forms.Form): def test_comparison(): compare_forms(FormA, FormA) - with pytest.raises(AssertionError, match="Forms have different field names"): + with pytest.raises(AssertionError, match="Found unexpected form fields"): test_different_fields(raiseit=True) with pytest.raises(AssertionError, match=r"Field mismatch \(myfield\)"): @@ -127,3 +127,24 @@ def test_doc_diff(): def test_different_whitespace(): compare_forms(DocForm, DocFormExtraWhitespace) + + +def test_equivalent_forms(): + class Form1(forms.Form): + myfield = forms.CharField(initial="whatever") + + class Form2(forms.Form): + myfield = forms.CharField(initial="whatever") + + compare_forms(Form1, Form2) + + +def test_differ_on_doc(): + class BasicForm(forms.Form): + myfield = forms.CharField(initial="whatever", help_text="what") + + class BasicFormDifferentDoc(forms.Form): + myfield = forms.CharField(initial="whatever", help_text="different") + + with pytest.raises(AssertionError, match="myfield.*"): + compare_forms(BasicForm, BasicFormDifferentDoc) diff --git a/turtle_shell/tests/utils.py b/turtle_shell/tests/utils.py index 1d9f47a..69ef694 100644 --- a/turtle_shell/tests/utils.py +++ b/turtle_shell/tests/utils.py @@ -6,12 +6,20 @@ def compare_form_field(name, actual, expected): """Compare important variables of two form fields""" - assert type(actual) == type(expected), f"Field type mismatch ({name})" + try: + assert type(actual) == type(expected) + except AssertionError as e: + # bare assert causes pytest rewrite, so we just add a bit around it + raise AssertionError(f"Field type mismatch ({name}): {e}") from e actual_vars = vars(actual) expected_vars = vars(expected) actual_vars.pop("widget", None) expected_vars.pop("widget", None) - assert actual_vars == expected_vars, f"Field mismatch ({name})" + try: + assert actual_vars == expected_vars + except AssertionError as e: + # bare assert causes pytest rewrite, so we just add a bit around it + raise AssertionError(f"Field mismatch ({name}): {e}") from e def compare_forms(actual: Type[forms.Form], expected: Type[forms.Form]): @@ -25,9 +33,23 @@ def compare_forms(actual: Type[forms.Form], expected: Type[forms.Form]): actual_fields = actual.declared_fields expected_fields = expected.declared_fields shared_keys = list(set(actual_fields.keys()) & set(expected_fields.keys())) + extra_keys = list(set(actual_fields.keys()) - set(expected_fields.keys())) + missing_keys = list(set(expected_fields.keys()) - set(actual_fields.keys())) for name in shared_keys: actual_field = actual_fields[name] expected_field = expected_fields[name] compare_form_field(name, actual_field, expected_field) - assert actual_fields.keys() == expected_fields.keys(), "Forms have different field names" - assert inspect.getdoc(actual) == inspect.getdoc(expected), "Forms have different docstrings" + assert not extra_keys, f"Found unexpected form fields:\n{extra_keys}" + assert not missing_keys, f"Expected fields missing:\n{missing_keys}" + try: + assert actual_fields.keys() == expected_fields.keys() + except AssertionError as e: + # bare assert causes pytest rewrite, so we just add a bit around it + raise AssertionError(f"Forms have different field names: {e}") from e + try: + assert inspect.getdoc(actual) == inspect.getdoc(expected) + except AssertionError as e: + print(repr(inspect.getdoc(actual))) + print(repr(inspect.getdoc(expected))) + # bare assert causes pytest rewrite, so we just add a bit around it + raise AssertionError(f"Forms have different docstrings: {e}") from e From be28cc9bd0d3f55e34960a7dcf1da73193c94009 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Sun, 11 Apr 2021 21:31:40 -0400 Subject: [PATCH 05/29] Getting empty value working woo --- turtle_shell/function_to_form.py | 3 +++ turtle_shell/tests/test_django_cli2ui.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index f1c773b..c1b9d18 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -141,6 +141,9 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: kwargs["required"] = True elif param.default is None: kwargs["required"] = False + # need this so that empty values get passed through to function correctly! + if 'empty_value' in inspect.signature(field_type).parameters: + kwargs['empty_value'] = None else: kwargs["required"] = False kwargs.setdefault("initial", param.default) diff --git a/turtle_shell/tests/test_django_cli2ui.py b/turtle_shell/tests/test_django_cli2ui.py index bd1f7e4..0339f08 100644 --- a/turtle_shell/tests/test_django_cli2ui.py +++ b/turtle_shell/tests/test_django_cli2ui.py @@ -89,10 +89,10 @@ class ExpectedFormForExampleFunc(forms.Form): text_arg = forms.CharField(help_text="should have a big text field", required=True) # TODO: different widget text_arg_with_default = forms.CharField(initial="something", help_text="should have big text field with something filled in", required=False) - undocumented_arg = forms.CharField(required=False) + undocumented_arg = forms.CharField(required=False, empty_value=None) enum_auto = forms.TypedChoiceField(choices=COLOR_CHOICES, required=True, help_text="should be choices with key names", coerce=Color) enum_auto_not_required = forms.TypedChoiceField(choices=COLOR_CHOICES, required=False, coerce=Color, - help_text="choice field not required") + help_text="choice field not required", empty_value=None) enum_auto_with_default = forms.TypedChoiceField(choices=COLOR_CHOICES, initial=Color.green.value, required=False, coerce=Color, help_text="choice field with entry selected") @@ -231,7 +231,6 @@ def myfunc(a: bool=True, b: str="whatever"): resp = execute_gql_and_get_input_json(myfunc, "mutation { executeMyfunc(input: {}) { result { inputJson }}}") assert resp == {"a": True, "b": "whatever"} -@pytest.mark.xfail def test_default_none(db): # defaults should be passed through From 31d4f6032ca682ad17b213a44bc184750b60fc0a Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Sun, 11 Apr 2021 21:44:19 -0400 Subject: [PATCH 06/29] Clean ups and such Clean stuff up a bit more --- turtle_shell/function_to_form.py | 141 +++++++++++++++++++---- turtle_shell/models.py | 15 ++- turtle_shell/tests/test_django_cli2ui.py | 104 ++++++++++++++--- turtle_shell/tests/utils.py | 12 +- 4 files changed, 219 insertions(+), 53 deletions(-) diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index c1b9d18..8543e4f 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -14,11 +14,14 @@ from dataclasses import dataclass from django import forms +from django.db.models import TextChoices from defopt import Parameter, signature, _parse_docstring from typing import Dict, Optional from typing import Type import pathlib +from . import utils + type2field_type = {int: forms.IntegerField, str: forms.CharField, bool: forms.BooleanField, Optional[bool]: forms.NullBooleanField, @@ -54,9 +57,21 @@ def function_to_form(func, *, config: dict = None, name: str=None) -> Type[forms fields = {} defaults = {} for parameter in sig.parameters.values(): - fields[parameter.name] = param_to_field(parameter, config) + field = param_to_field(parameter, config) + fields[parameter.name] = field if parameter.default is not Parameter.empty: defaults[parameter.name] = parameter.default + if isinstance(field, forms.TypedChoiceField): + field._parameter_name = parameter.name + field._func_name = name + if parameter.default and parameter.default is not Parameter.empty: + print(field.choices) + for potential_default in [parameter.default.name, parameter.default.value]: + if any(potential_default == x[0] for x in field.choices): + defaults[parameter.name] = potential_default + break + else: + raise ValueError(f"Cannot figure out how to assign default for {parameter.name}: {parameter.default}") fields["__doc__"] = re.sub("\n+", "\n", _parse_docstring(inspect.getdoc(func)).text) form_name = "".join(part.capitalize() for part in func.__name__.split("_")) @@ -103,6 +118,54 @@ def get_type_from_annotation(param: Parameter): return param.annotation +@dataclass +class Coercer: + """Wrapper so that we handle implicit string conversion of enum types :(""" + enum_type: object + by_attribute: bool = False + + def __call__(self, value): + print(f"COERCE: {self} {value}") + try: + resp = self._call(value) + print(f"COERCED TO: {self} {value} => {resp}") + return resp + except Exception as e: + import traceback + print(f"FAILED TO COERCE {repr(value)}({value})") + traceback.print_exc() + raise + def _call(self, value): + if value and isinstance(value, self.enum_type): + print("ALREADY INSTANCE") + return value + if self.by_attribute: + print("BY ATTRIBUTE") + return getattr(self.enum_type, value) + try: + print("BY __call__") + resp = self.enum_type(value) + print(f"RESULT: {resp} ({repr(resp)})") + return resp + except ValueError as e: + import traceback + traceback.print_exc() + try: + print("BY int coerced __call__") + return self.enum_type(int(value)) + except ValueError as f: + # could not coerce to int :( + pass + if isinstance(value, str): + # fallback to some kind of name thing if necesary + try: + return getattr(self.enum_type, value) + except AttributeError: + pass + raise e from e + assert False, "Should not get here" + + def param_to_field(param: Parameter, config: dict = None) -> forms.Field: """Convert a specific arg to a django form field. @@ -121,22 +184,59 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: # e.g. stupid generic type stuff pass if is_enum_class: + utils.EnumRegistry.register(kind) field_type = forms.TypedChoiceField - kwargs["coerce"] = kind - kwargs["choices"] = [(member.name, member.value) for member in kind] - # coerce back - if isinstance(param.default, kind): - kwargs["initial"] = param.default.value + kwargs.update(make_enum_kwargs(param=param, kind=kind)) else: - field_type = all_types.get(param.annotation, all_types.get(kind)) - for k, v in all_types.items(): - if field_type: - break - if inspect.isclass(k) and issubclass(kind, k) or k == kind: - field_type = v - break - if not field_type: - raise ValueError(f"Field {param.name}: Unknown field type: {param.annotation}") + field_type = get_for_param_by_type(all_types, param=param, kind=kind) + if not field_type: + raise ValueError(f"Field {param.name}: Unknown field type: {param.annotation}") + # do not overwrite kwargs if already specified + kwargs = {**extra_kwargs(field_type, param), **kwargs} + if field_type == forms.BooleanField and param.default is None: + field_type = forms.NullBooleanField + + widget = get_for_param_by_type(widgets, param=param, kind=kind) + if widget: + kwargs['widget'] = widget + return field_type(**kwargs) + + +def make_enum_kwargs(kind, param): + kwargs = {} + if all(isinstance(member.value, int) for member in kind): + kwargs["choices"] = TextChoices(f'{kind.__name__}Enum', {member.name: (member.name, member.name) for + member in kind}).choices + kwargs["coerce"] = Coercer(kind, by_attribute=True) + else: + # we set up all the kinds of entries to make it a bit easier to do the names and the + # values... + kwargs["choices"] = TextChoices(f'{kind.__name__}Enum', dict([(member.name, (str(member.value), + member.name)) for member in kind] + [(str(member.value), (member.name, member.name)) for + member in kind])).choices + kwargs["coerce"] = Coercer(kind) + # coerce back + if isinstance(param.default, kind): + kwargs["initial"] = param.default.value + return kwargs + + +def get_for_param_by_type(dct, *, param, kind): + """Grab the appropriate element out of dict based on param type. + + Ordering: + 1. param.name (i.e., something custom specified by user) + 2. param.annotation + 3. underlying type if typing.Optional + """ + if elem := dct.get(param.name, dct.get(param.annotation, dct.get(kind))): + return elem + for k, v in dct.items(): + if inspect.isclass(k) and issubclass(kind, k) or k == kind: + return v + +def extra_kwargs(field_type, param): + kwargs = {} if param.default is Parameter.empty: kwargs["required"] = True elif param.default is None: @@ -149,15 +249,6 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: kwargs.setdefault("initial", param.default) if param.doc: kwargs["help_text"] = param.doc - for k, v in widgets.items(): - if isinstance(k, str): - if param.name == k: - kwargs["widget"] = v - break - continue - if issubclass(k, param.annotation): - kwargs["widget"] = v - break - return field_type(**kwargs) + return kwargs diff --git a/turtle_shell/models.py b/turtle_shell/models.py index dbc2a29..9578f0d 100644 --- a/turtle_shell/models.py +++ b/turtle_shell/models.py @@ -1,6 +1,7 @@ -from django.db import models +from django.db import models, transaction from django.urls import reverse from django.conf import settings +from turtle_shell import utils import uuid class CaughtException(Exception): @@ -19,9 +20,11 @@ class ResultJSONEncodeException(CaughtException): class ExecutionResult(models.Model): uuid = models.UUIDField(primary_key=True, unique=True, editable=False, default=uuid.uuid4) func_name = models.CharField(max_length=512, editable=False) - input_json = models.JSONField() - output_json = models.JSONField(default=dict, null=True) - error_json = models.JSONField(default=dict, null=True) + input_json = models.JSONField(encoder=utils.EnumAwareEncoder, decoder=utils.EnumAwareDecoder) + output_json = models.JSONField(default=dict, null=True, encoder=utils.EnumAwareEncoder, + decoder=utils.EnumAwareDecoder) + error_json = models.JSONField(default=dict, null=True, encoder=utils.EnumAwareEncoder, + decoder=utils.EnumAwareDecoder) class ExecutionStatus(models.TextChoices): CREATED = 'CREATED', 'Created' @@ -54,7 +57,9 @@ def execute(self): try: self.output_json = result self.status = self.ExecutionStatus.DONE - self.save() + # allow ourselves to save again externally + with transaction.atomic(): + self.save() except TypeError as e: self.error_json = {"type": type(e).__name__, "message": str(e)} msg = f"Failed on {self.func_name} ({type(e).__name__})" diff --git a/turtle_shell/tests/test_django_cli2ui.py b/turtle_shell/tests/test_django_cli2ui.py index 0339f08..c703d4d 100644 --- a/turtle_shell/tests/test_django_cli2ui.py +++ b/turtle_shell/tests/test_django_cli2ui.py @@ -7,6 +7,8 @@ from .utils import compare_form_field, compare_forms from turtle_shell.function_to_form import param_to_field from turtle_shell.function_to_form import function_to_form +from turtle_shell.function_to_form import Coercer +from turtle_shell import utils import turtle_shell from defopt import Parameter from typing import Optional @@ -19,7 +21,7 @@ class Color(enum.Enum): yellow = enum.auto() -COLOR_CHOICES = [(e.name, e.value) for e in Color] +COLOR_CHOICES = [(e.name, e.name) for e in Color] class Flag(enum.Enum): @@ -90,17 +92,17 @@ class ExpectedFormForExampleFunc(forms.Form): # TODO: different widget text_arg_with_default = forms.CharField(initial="something", help_text="should have big text field with something filled in", required=False) undocumented_arg = forms.CharField(required=False, empty_value=None) - enum_auto = forms.TypedChoiceField(choices=COLOR_CHOICES, required=True, help_text="should be choices with key names", coerce=Color) - enum_auto_not_required = forms.TypedChoiceField(choices=COLOR_CHOICES, required=False, coerce=Color, + enum_auto = forms.TypedChoiceField(choices=COLOR_CHOICES, required=True, help_text="should be choices with key names", coerce=Coercer(Color, by_attribute=True)) + enum_auto_not_required = forms.TypedChoiceField(choices=COLOR_CHOICES, required=False, coerce=Coercer(Color, by_attribute=True), help_text="choice field not required", empty_value=None) enum_auto_with_default = forms.TypedChoiceField(choices=COLOR_CHOICES, initial=Color.green.value, required=False, - coerce=Color, help_text="choice field with entry selected") + coerce=Coercer(Color, by_attribute=True), help_text="choice field with entry selected") enum_str = forms.TypedChoiceField(choices=[("is_apple", "is_apple"), ("is_banana", - "is_banana")], required=True, coerce=Flag, help_text="should be choices with string values") + "is_banana")], required=True, coerce=Coercer(Flag), help_text="should be choices with string values") enum_str_with_default = forms.TypedChoiceField(choices=[("is_apple", "is_apple"), ("is_banana", "is_banana")], required=False, initial=Flag.is_apple.value, - coerce=Flag) + coerce=Coercer(Flag)) def test_compare_complex_example(db): @@ -145,6 +147,10 @@ def _make_parameter(name, annotation, doc="", **kwargs): _make_parameter("str_doc_falsey", str, "some doc", default=""), forms.CharField(initial="", required=False, help_text="some doc"), ), + ( + _make_parameter("bool_default_none", bool, "some doc", default=None), + forms.NullBooleanField(required=False, help_text="some doc") + ), ( _make_parameter("bool_falsey", bool, "some doc", default=False), forms.BooleanField(required=False, initial=False, help_text="some doc"), @@ -158,13 +164,28 @@ def _make_parameter(name, annotation, doc="", **kwargs): forms.BooleanField(required=True, help_text="some doc"), ), ( - _make_parameter("enum_auto", Color), - forms.TypedChoiceField(coerce=Color, choices=COLOR_CHOICES), + _make_parameter("optional_bool", Optional[bool], default=True), + forms.NullBooleanField(initial=True, required=False) ), + ( + _make_parameter("optional_bool", Optional[bool], default=None), + forms.NullBooleanField(initial=None, required=False) + ), + ], + ids=lambda x: x.name if hasattr(x, "name") else x, +) +def test_convert_arg(arg, expected): + field = param_to_field(arg) + compare_form_field(arg.name, field, expected) + + +@pytest.mark.parametrize( + "arg,expected", + [ ( _make_parameter("enum_auto_default", Color, "another doc", default=Color.green), forms.TypedChoiceField( - coerce=Color, + coerce=Coercer(Color, by_attribute=True), initial=Color.green.value, choices=COLOR_CHOICES, required=False, @@ -173,22 +194,18 @@ def _make_parameter(name, annotation, doc="", **kwargs): ), ( _make_parameter("enum_str", Flag), - forms.TypedChoiceField(coerce=Flag, choices=FLAG_CHOICES), + forms.TypedChoiceField(coerce=Coercer(Flag), choices=FLAG_CHOICES), ), ( _make_parameter("enum_str_default", Flag, default=Flag.is_apple), forms.TypedChoiceField( - coerce=Flag, initial="is_apple", choices=FLAG_CHOICES, required=False + coerce=Coercer(Flag), initial="is_apple", choices=FLAG_CHOICES, required=False ), ), - ( - _make_parameter("optional_bool", Optional[bool], default=True), - forms.NullBooleanField(initial=True, required=False) - ), ], ids=lambda x: x.name if hasattr(x, "name") else x, ) -def test_convert_arg(arg, expected): +def test_enum_convert_arg(arg, expected): field = param_to_field(arg) compare_form_field(arg.name, field, expected) @@ -212,14 +229,20 @@ def test_validators(): pass -def execute_gql_and_get_input_json(func, gql): +def execute_gql(func, gql): registry = turtle_shell.get_registry() registry.clear() registry.add(func) result = registry.schema.execute(gql) + return result + +def execute_gql_and_get_input_json(func, gql): + result = execute_gql(func, gql) data = result.data assert not result.errors - return json.loads(list(data.values())[0]["result"]["inputJson"]) + result_from_response = list(data.values())[0]["result"] + assert result_from_response + return json.loads(result_from_response["inputJson"]) # data = json.loads(result["data"]["result"]["inputJson"]) # return data @@ -231,6 +254,22 @@ def myfunc(a: bool=True, b: str="whatever"): resp = execute_gql_and_get_input_json(myfunc, "mutation { executeMyfunc(input: {}) { result { inputJson }}}") assert resp == {"a": True, "b": "whatever"} +def test_enum_preservation(db): + class ReadType(enum.Enum): + fastq = enum.auto() + bam = enum.auto() + + def func(read_type: ReadType=ReadType.fastq): + return read_type + + input_json = execute_gql_and_get_input_json(func, 'mutation { executeFunc(input: {readType: BAM}) { result { inputJson }}}') + assert input_json == {"read_type": utils.EnumRegistry.to_json_repr(ReadType.bam)} + assert input_json["read_type"]["__enum__"]["name"] == "bam" + + input_json = execute_gql_and_get_input_json(func, "mutation { executeFunc(input: {}) { result { inputJson }}}") + assert input_json == {"read_type": utils.EnumRegistry.to_json_repr(ReadType.fastq)} + assert input_json["read_type"]["__enum__"]["name"] == "fastq" + def test_default_none(db): # defaults should be passed through @@ -262,3 +301,32 @@ def my_func(*, a: bool, b: str): def test_exceptions(parameter, exception_type, msg_regex): with pytest.raises(exception_type, match=msg_regex): param_to_field(parameter) + + +def test_coercer(): + class AutoEnum(enum.Enum): + whatever = enum.auto() + another = enum.auto() + + class StringlyIntEnum(enum.Enum): + val1 = '1' + val2 = '2' + + assert Coercer(AutoEnum)(AutoEnum.whatever.value) == AutoEnum.whatever + assert Coercer(AutoEnum)(str(AutoEnum.whatever.value)) == AutoEnum.whatever + assert Coercer(StringlyIntEnum)('1') == StringlyIntEnum('1') + with pytest.raises(ValueError): + Coercer(StringlyIntEnum)(1) + + +def test_rendering_enum_with_mixed_type(db): + class MiscStringEnum(enum.Enum): + whatever = 'bobiswhatever' + mish = 'dish' + defa = 'default yeah' + + def func(s: MiscStringEnum=MiscStringEnum.defa): + return s + input_json = execute_gql_and_get_input_json(func, "mutation { executeFunc(input: {}) { result { inputJson }}}") + input_json2 = execute_gql_and_get_input_json(func, "mutation { executeFunc(input: {s: DEFAULT_YEAH}) { result { inputJson }}}") + assert input_json == input_json2 diff --git a/turtle_shell/tests/utils.py b/turtle_shell/tests/utils.py index 69ef694..fd207c4 100644 --- a/turtle_shell/tests/utils.py +++ b/turtle_shell/tests/utils.py @@ -13,8 +13,9 @@ def compare_form_field(name, actual, expected): raise AssertionError(f"Field type mismatch ({name}): {e}") from e actual_vars = vars(actual) expected_vars = vars(expected) - actual_vars.pop("widget", None) - expected_vars.pop("widget", None) + for k in ("widget", "_func_name", "_parameter_name"): + actual_vars.pop(k, None) + expected_vars.pop(k, None) try: assert actual_vars == expected_vars except AssertionError as e: @@ -32,9 +33,10 @@ def compare_forms(actual: Type[forms.Form], expected: Type[forms.Form]): """ actual_fields = actual.declared_fields expected_fields = expected.declared_fields - shared_keys = list(set(actual_fields.keys()) & set(expected_fields.keys())) - extra_keys = list(set(actual_fields.keys()) - set(expected_fields.keys())) - missing_keys = list(set(expected_fields.keys()) - set(actual_fields.keys())) + excluded_keys = {'_func_name', '_parameter_name'} + shared_keys = list(set(actual_fields.keys()) & set(expected_fields.keys()) - excluded_keys) + extra_keys = list(set(actual_fields.keys()) - set(expected_fields.keys()) - excluded_keys) + missing_keys = list(set(expected_fields.keys()) - set(actual_fields.keys()) - excluded_keys) for name in shared_keys: actual_field = actual_fields[name] expected_field = expected_fields[name] From 8af8d8e424df7351c3b10e7869c3ba018ef4c784 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Mon, 12 Apr 2021 09:17:44 -0400 Subject: [PATCH 07/29] Stuff seems to mostly work yay Set up Do stuff --- turtle_shell/README.rst | 98 ++++++++----- turtle_shell/function_to_form.py | 6 +- turtle_shell/graphene_adapter.py | 133 ++++++++++++++++++ turtle_shell/graphene_adapter_jsonstring.py | 35 +++++ .../migrations/0002_auto_20210411_1045.py | 33 +++++ .../migrations/0003_auto_20210411_1104.py | 31 ++++ .../migrations/0004_auto_20210411_1223.py | 23 +++ turtle_shell/models.py | 3 + .../turtle_shell/executionresult_create.html | 9 ++ .../templates/turtle_shell/overview.html | 1 + turtle_shell/tests/test_utils.py | 24 ++++ turtle_shell/utils.py | 62 ++++++++ 12 files changed, 424 insertions(+), 34 deletions(-) create mode 100644 turtle_shell/graphene_adapter.py create mode 100644 turtle_shell/graphene_adapter_jsonstring.py create mode 100644 turtle_shell/migrations/0002_auto_20210411_1045.py create mode 100644 turtle_shell/migrations/0003_auto_20210411_1104.py create mode 100644 turtle_shell/migrations/0004_auto_20210411_1223.py create mode 100644 turtle_shell/templates/turtle_shell/executionresult_create.html create mode 100644 turtle_shell/tests/test_utils.py create mode 100644 turtle_shell/utils.py diff --git a/turtle_shell/README.rst b/turtle_shell/README.rst index f3f5e33..9f0d404 100644 --- a/turtle_shell/README.rst +++ b/turtle_shell/README.rst @@ -1,15 +1,6 @@ Function to Form View ===================== -CURRENTLY BROKEN STUFF: - -1. Graphql view does not respect defaults :( (e.g., find_root_cause comes through as false :() -2. Needs testing -3. Needs lots of testing - -LIMITATIONS: - -1. Boolean fields always are passed in as False. Motivation ---------- @@ -20,11 +11,17 @@ Motivation 4. You want people to have website goodness (deep-linking, record of actions, easy on-boarding, etc) 5. Composing and/or ability to long-poll endpoints seems cool to you. +REMAINING WORK: + +1. Ability to do asynchronous executions (this is basically all set up) +2. Better UI on output and/or ability to have structured graphql output for nicer APIs + Maybe some kind of output serializer? See end for some ideas. + How does it work? ----------------- -This lil' old library converts _your_ function with annotations into a ✨Django Form✨ (that calls the function on ``form_valid``!) +This lil' old library converts _your_ function with annotations into a ✨Django Form✨ and a graphql view. It leverages some neat features of defopt under the hood so that a function like this: @@ -52,6 +49,16 @@ Becomes this awesome form! +Why not FastAPI? +---------------- + +This is a great point! I didn't see it before I started. +Using Django provides: + +0. FRONT END! -> key for non-technical users +1. Easy ability to add in authentication/authorization (granted FastAPI has this) +2. Literally didn't see it and we know django better + Overall gist ------------ @@ -87,15 +94,15 @@ Example Implementation executions.py:: - from easy_execute import ExecutionWrapper + import turtle_shell from my_util_scripts import find_root_cause, summarize_issue, error_summary - Registry = ExecutionWrapper() + Registry = turtle_shell.get_registry() - FindRootCause = Registry.wrap(find_root_cause) - SummarizeIssue = Registry.wrap(summarize_issue) - ErrorSummary = Registry.wrap(error_summary) + FindRootCause = Registry.add(find_root_cause) + SummarizeIssue = Registry.add(summarize_issue) + ErrorSummary = Registry.add(error_summary) @@ -130,12 +137,17 @@ Of course you can also customize further:: views:: - from . import executions + import turtle_shell + + Registry = turtle_shell.get_registry() - class FindRootCauseList(executions.FindRootCause.list_view()): + class FindRootCauseList(Registry.get(find_root_cause).list_view()): template_name = "list-root-cause.html" - class FindRootCauseDetail(executions.FindRootCause.detail_view()): + def get_context_data(self): + # do some processing here - yay! + + class FindRootCauseDetail(Registry.get(find_root_cause).detail_view()): template_name = "detail-root-cause.html" These use the generic django views under the hood. @@ -145,6 +157,7 @@ What's missing from this idea - granular permissions (gotta think about nice API for this) - separate tables for different objects. +- some kind of nicer serializer Using the library ----------------- @@ -170,14 +183,12 @@ Every function gets a generic output:: mutation { dxFindRootCause(input: {job_id: ..., project: ...}) { uuid: str - execution { - status: String? - exitCode: Int - successful: Bool - } - rawOutput { - stderr: String? - stdout: String # often JSON serializable + result { + status: STATUS + uuid: UUID! + inputJson: String! + outputJson: String? # often JSON serializable + errorJson: String? } } errors: Optional { @@ -239,11 +250,34 @@ who were not as command line savvy. 2. Expose utility functions as forms for users -Customizing the forms ---------------------- +How output serializers might look +--------------------------------- + +1. One cool idea would just be to automatically convert to and from attrs classes using `cattrs` to customize output. (little more flexible in general) +2. Could just return django models that get persisted (advantage is a bit easier to see old executions) +3. Use pydantic to have some nice structure on output types :) + +Attrs classes +^^^^^^^^^^^^^ + +Concept: + 1. attr_to_graphene => convert attrs classes into nested graphene type. Handles resolving those fields from result + 2. cattr.structure/cattr.unstructure to marshal to and from JSON + +Pros: + +* Easy to represent deeply nested contents +* Do not need to save to DB + +Cons: + +* Reimplement a lot of the graphene django work :( + + +Pydantic classes +^^^^^^^^^^^^^^^^ -First - you can pass a config dictionary to ``function_to_form`` to tell it to -use particular widgets for fields or how to construct a form field for your custom type ( -as a callable that takes standard field keyword arguments). +Better support for unmarshalling +works with fast api as well -You can also subclass the generated form object to add your own ``clean_*`` methods or more complex validation - yay! +https://pydantic-docs.helpmanual.io/usage/models/#data-conversion diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index 8543e4f..196edb0 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -32,11 +32,13 @@ class _Function: func: callable name: str form_class: object + doc: str @classmethod def from_function(cls, func, *, name): - from turtle_shell.function_to_form import function_to_form - return cls(func=func, name=name, form_class=function_to_form(func, name=name)) + form_class = function_to_form(func, name=name) + return cls(func=func, name=name, form_class=form_class, + doc=form_class.__doc__) def doc_mapping(str) -> Dict[str, str]: diff --git a/turtle_shell/graphene_adapter.py b/turtle_shell/graphene_adapter.py new file mode 100644 index 0000000..2e4ef6e --- /dev/null +++ b/turtle_shell/graphene_adapter.py @@ -0,0 +1,133 @@ +import json +import graphene +from graphene_django.forms.mutation import DjangoFormMutation +from graphene_django import DjangoObjectType +from graphene_django.filter import DjangoFilterConnectionField +from graphene import relay +from . import models +from graphene_django.forms import converter as graphene_django_converter +from django import forms +from . import utils +# PATCH IT GOOD! +import turtle_shell.graphene_adapter_jsonstring + +_seen_names: set = set() + +# class FakeJSONModule: +# @classmethod +# def loads(self, *a, **k): +# return json.loads(*a, **k) +# +# @classmethod +# def dumps(self, *a, **k): +# k['cls'] = utils.EnumAwareEncoder +# return json.dumps(*a, **k) +# +# @classmethod +# def dump(self, *a, **k): +# k['cls'] = utils.EnumAwareEncoder +# return json.dump(*a, **k) +# +# import graphene_django.views +# import graphene.types.json +# +# graphene_django.views.json = FakeJSONModule +# graphene.types.json.json = FakeJSONModule + + +@graphene_django_converter.convert_form_field.register(forms.TypedChoiceField) +def convert_form_field_to_choice(field): + # TODO: this should really be ported back to graphene django + from graphene_django.converter import convert_choice_field_to_enum + name = full_name = f'{field._func_name}{field._parameter_name}' + index = 0 + while full_name in _seen_names: + index += 1 + full_name = f'{name}{index}' + EnumCls = convert_choice_field_to_enum(field, name=full_name) + print(EnumCls, getattr(EnumCls, "BAM", None)) + converted = EnumCls( + description=field.help_text, required=field.required + ) + _seen_names.add(full_name) + + return converted + +class ExecutionResult(DjangoObjectType): + class Meta: + model = models.ExecutionResult + interfaces = (relay.Node,) + filter_fields = { + "func_name": ["exact"], + "uuid": ["exact"], + } + fields = [ + "uuid", + "func_name", + "status", + "input_json", + "output_json", + "error_json", + "created", + "modified", + # TODO: will need this to be set up better + # "user" + ] + + +def func_to_graphene_form_mutation(func_object): + form_class = func_object.form_class + defaults = getattr(func_object.form_class, "_input_defaults", None) or {} + + class Meta: + form_class = func_object.form_class + + @classmethod + def mutate_and_get_payload(cls, root, info, **input): + """Set defaults from function in place!""" + input = {**defaults, **input} + print(f"MUTATE GET PAYLOAD {input} {repr(input.get('read_type'))} {type(input.get('read_type'))}") + form = cls.get_form(root, info, **input) + if not form.is_valid(): + print(form.errors) + try: + return super(DefaultOperationMutation, cls).mutate_and_get_payload(root, info, **input) + except Exception as e: + import traceback + traceback.print_exc() + raise + + @classmethod + def perform_mutate(cls, form, info): + obj = form.save() + obj.execute() + obj.save() + # TODO: make errors show up nicely + return cls(errors=[], result=obj) + + # TODO: figure out if name can be customized in class + mutation_name = f'{form_class.__name__}Mutation' + DefaultOperationMutation = type(mutation_name, (DjangoFormMutation,), ({"Meta": Meta, + "perform_mutate": perform_mutate, "result": graphene.Field(ExecutionResult), "__doc__": + f'Mutation form for {form_class.__name__}.\n{form_class.__doc__}', + "mutate_and_get_payload": mutate_and_get_payload})) + return DefaultOperationMutation + +def schema_for_registry(registry): + # TODO: make this more flexible! + class Query(graphene.ObjectType): + execution_results = DjangoFilterConnectionField(ExecutionResult) + execution_result = graphene.Field(ExecutionResult, uuid=graphene.String()) + + def resolve_execution_result(cls, info, uuid): + try: + return models.ExecutionResult.objects.get(pk=uuid) + except models.ExecutionResult.DoesNotExist: + pass + mutation_fields = {} + for func_obj in registry.func_name2func.values(): + mutation = func_to_graphene_form_mutation(func_obj) + mutation_fields[f'execute_{func_obj.name}'] = mutation.Field() + Mutation = type("Mutation", (graphene.ObjectType,), mutation_fields) + return graphene.Schema(query=Query, mutation=Mutation) + diff --git a/turtle_shell/graphene_adapter_jsonstring.py b/turtle_shell/graphene_adapter_jsonstring.py new file mode 100644 index 0000000..6d80e2a --- /dev/null +++ b/turtle_shell/graphene_adapter_jsonstring.py @@ -0,0 +1,35 @@ +"""Patch graphene django's JSONString implementation so we can use a custom encoder""" +#Port over of graphene's JSON string to allow using a custom encoder...sigh +import json + +from graphql.language import ast + +from graphene.types.scalars import Scalar + + +class CustomEncoderJSONString(Scalar): + """ + Allows use of a JSON String for input / output from the GraphQL schema. + + Use of this type is *not recommended* as you lose the benefits of having a defined, static + schema (one of the key benefits of GraphQL). + """ + + @staticmethod + def serialize(dt): + from turtle_shell import utils + return json.dumps(dt, cls=utils.EnumAwareEncoder) + + @staticmethod + def parse_literal(node): + if isinstance(node, ast.StringValue): + return json.loads(node.value) + + @staticmethod + def parse_value(value): + return json.loads(value) + + +from graphene_django import converter + +converter.JSONString = CustomEncoderJSONString diff --git a/turtle_shell/migrations/0002_auto_20210411_1045.py b/turtle_shell/migrations/0002_auto_20210411_1045.py new file mode 100644 index 0000000..c0c30de --- /dev/null +++ b/turtle_shell/migrations/0002_auto_20210411_1045.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.8 on 2021-04-11 17:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('turtle_shell', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='executionresult', + name='error_json', + field=models.JSONField(default=None), + ), + migrations.AlterField( + model_name='executionresult', + name='func_name', + field=models.CharField(editable=False, max_length=512), + ), + migrations.AlterField( + model_name='executionresult', + name='output_json', + field=models.JSONField(default=None), + ), + migrations.AlterField( + model_name='executionresult', + name='status', + field=models.CharField(choices=[('CREATED', 'Created'), ('RUNNING', 'Running'), ('DONE', 'Done'), ('ERRORED', 'Errored'), ('JSON_ERROR', 'Result could not be coerced to JSON')], default='CREATED', max_length=10), + ), + ] diff --git a/turtle_shell/migrations/0003_auto_20210411_1104.py b/turtle_shell/migrations/0003_auto_20210411_1104.py new file mode 100644 index 0000000..2fbcd90 --- /dev/null +++ b/turtle_shell/migrations/0003_auto_20210411_1104.py @@ -0,0 +1,31 @@ +# Generated by Django 3.1.8 on 2021-04-11 18:04 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('turtle_shell', '0002_auto_20210411_1045'), + ] + + operations = [ + migrations.AddField( + model_name='executionresult', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='executionresult', + name='error_json', + field=models.JSONField(default=dict), + ), + migrations.AlterField( + model_name='executionresult', + name='output_json', + field=models.JSONField(default=dict), + ), + ] diff --git a/turtle_shell/migrations/0004_auto_20210411_1223.py b/turtle_shell/migrations/0004_auto_20210411_1223.py new file mode 100644 index 0000000..f20328a --- /dev/null +++ b/turtle_shell/migrations/0004_auto_20210411_1223.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.8 on 2021-04-11 19:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('turtle_shell', '0003_auto_20210411_1104'), + ] + + operations = [ + migrations.AlterField( + model_name='executionresult', + name='error_json', + field=models.JSONField(default=dict, null=True), + ), + migrations.AlterField( + model_name='executionresult', + name='output_json', + field=models.JSONField(default=dict, null=True), + ), + ] diff --git a/turtle_shell/models.py b/turtle_shell/models.py index 9578f0d..5f81a64 100644 --- a/turtle_shell/models.py +++ b/turtle_shell/models.py @@ -3,6 +3,7 @@ from django.conf import settings from turtle_shell import utils import uuid +import cattr class CaughtException(Exception): """An exception that was caught and saved. Generally don't need to rollback transaction with @@ -55,6 +56,8 @@ def execute(self): self.save() raise CaughtException(f"Failed on {self.func_name} ({type(e).__name__})", e) from e try: + if not isinstance(result, (dict, str, tuple)): + result = cattr.unstructure(result) self.output_json = result self.status = self.ExecutionStatus.DONE # allow ourselves to save again externally diff --git a/turtle_shell/templates/turtle_shell/executionresult_create.html b/turtle_shell/templates/turtle_shell/executionresult_create.html new file mode 100644 index 0000000..4619c35 --- /dev/null +++ b/turtle_shell/templates/turtle_shell/executionresult_create.html @@ -0,0 +1,9 @@ +{% extends 'base.html' %} +{% load crispy_forms_tags %} + +{% block content %} +

New Execution for {{func_name}}

+{% crispy form %} +{% endblock content %} + + diff --git a/turtle_shell/templates/turtle_shell/overview.html b/turtle_shell/templates/turtle_shell/overview.html index 452577f..af5d18c 100644 --- a/turtle_shell/templates/turtle_shell/overview.html +++ b/turtle_shell/templates/turtle_shell/overview.html @@ -6,6 +6,7 @@
  • {{elem.name}} ( Create New Execution )

    +{% if elem.doc %}

    {{elem.doc}}

    {% endif %} {% endfor %} {% endblock content %} diff --git a/turtle_shell/tests/test_utils.py b/turtle_shell/tests/test_utils.py new file mode 100644 index 0000000..564666a --- /dev/null +++ b/turtle_shell/tests/test_utils.py @@ -0,0 +1,24 @@ +from turtle_shell import utils +import enum +import json +import pytest + +def test_json_encoder_not_registered(): + class MyEnum(enum.Enum): + a = enum.auto() + + with pytest.raises(TypeError, match="has not been registered"): + json.dumps(MyEnum.a, cls=utils.EnumAwareEncoder) + +def test_json_encoder_registered(): + class MyOtherEnum(enum.Enum): + a = enum.auto() + + utils.EnumRegistry.register(MyOtherEnum) + + original = {"val": MyOtherEnum.a} + s = json.dumps(original, cls=utils.EnumAwareEncoder) + assert '"__enum__"' in s + round_trip = json.loads(s, cls=utils.EnumAwareDecoder) + assert round_trip == original + diff --git a/turtle_shell/utils.py b/turtle_shell/utils.py new file mode 100644 index 0000000..be6e6e4 --- /dev/null +++ b/turtle_shell/utils.py @@ -0,0 +1,62 @@ +import json +import enum +from collections import defaultdict +from django.core.serializers.json import DjangoJSONEncoder + +class EnumRegistry: + # URP - global! :( + _registered_enums = defaultdict(dict) + + @classmethod + def register(cls, enum_class): + cls._registered_enums[enum_class.__module__][enum_class.__qualname__] = enum_class + + @classmethod + def has_enum(cls, enum_class): + try: + return cls._registered_enums[enum_class.__module__][enum_class.__qualname__] + except KeyError: + return None + + @classmethod + def to_json_repr(cls, enum_member): + if not cls.has_enum(type(enum_member)): + raise TypeError(f"Enum type {type(enum_member)} has not been registered and can't be serialized :(") + enum_class = type(enum_member) + return {"__enum__": {"__type__": [enum_class.__module__, enum_class.__qualname__], "name": enum_member.name, "value": enum_member.value}} + + @classmethod + def from_json_repr(cls, json_obj): + if "__enum__" not in json_obj: + raise ValueError("Enums must be represented by __enum__ key") + try: + type_data = json_obj["__enum__"]["__type__"] + try: + enum_class = cls._registered_enums[type_data[0]][type_data[1]] + except KeyError: + raise ValueError(f"Looks like enum {type_data} is not registered :(") + return enum_class(json_obj["__enum__"]["value"]) + except (KeyError, ValueError) as e: + raise ValueError(f"Invalid enum representation in JSON:: {type(e).__name__}: {e}") + + @classmethod + def object_hook(cls, dct): + if '__enum__' not in dct: + return dct + return cls.from_json_repr(dct) + + +class EnumAwareEncoder(DjangoJSONEncoder): + def default(self, o, **k): + if isinstance(o, enum.Enum): + return EnumRegistry.to_json_repr(o) + else: + super().default(o, **k) + +class EnumAwareDecoder(json.JSONDecoder): + def __init__(self, *a, **k): + k.setdefault('object_hook', self.object_hook) + super().__init__(*a, **k) + + def object_hook(self, dct): + return EnumRegistry.object_hook(dct) From 68471932dc8aca3924680df41ffac406e026c301 Mon Sep 17 00:00:00 2001 From: Jeff Tratner Date: Mon, 12 Apr 2021 11:32:59 -0400 Subject: [PATCH 08/29] Restructure tests yay Set up restore form - yay More set up and tests and such --- turtle_shell/README.rst | 42 ++++++---- turtle_shell/__init__.py | 6 +- turtle_shell/fake_pydantic_adpater.py | 5 ++ turtle_shell/function_to_form.py | 19 +++-- turtle_shell/graphene_adapter.py | 19 ++++- turtle_shell/models.py | 25 +++++- turtle_shell/pydantic_adapter.py | 78 ++++++++++++++++++ .../turtle_shell/executionresult_create.html | 1 + .../turtle_shell/executionresult_detail.html | 42 ++++++---- .../turtle_shell/executionresult_list.html | 30 +++++-- .../executionresult_summaryrow.html | 11 +++ .../templatetags/pydantic_to_table.py | 58 +++++++++++++ turtle_shell/tests/test_django_cli2ui.py | 82 ------------------- turtle_shell/tests/test_graphene_adapter.py | 68 +++++++++++++++ turtle_shell/tests/test_pydantic_adapter.py | 80 ++++++++++++++++++ turtle_shell/tests/utils.py | 21 +++++ turtle_shell/views.py | 4 +- 17 files changed, 452 insertions(+), 139 deletions(-) create mode 100644 turtle_shell/fake_pydantic_adpater.py create mode 100644 turtle_shell/pydantic_adapter.py create mode 100644 turtle_shell/templates/turtle_shell/executionresult_summaryrow.html create mode 100644 turtle_shell/templatetags/pydantic_to_table.py create mode 100644 turtle_shell/tests/test_graphene_adapter.py create mode 100644 turtle_shell/tests/test_pydantic_adapter.py diff --git a/turtle_shell/README.rst b/turtle_shell/README.rst index 9f0d404..2a714a2 100644 --- a/turtle_shell/README.rst +++ b/turtle_shell/README.rst @@ -16,6 +16,7 @@ REMAINING WORK: 1. Ability to do asynchronous executions (this is basically all set up) 2. Better UI on output and/or ability to have structured graphql output for nicer APIs Maybe some kind of output serializer? See end for some ideas. +3. Help graphene-django release a version based on graphql-core so we can use newer graphene-pydantic :P How does it work? ----------------- @@ -49,16 +50,7 @@ Becomes this awesome form! -Why not FastAPI? ----------------- - -This is a great point! I didn't see it before I started. -Using Django provides: - -0. FRONT END! -> key for non-technical users -1. Easy ability to add in authentication/authorization (granted FastAPI has this) -2. Literally didn't see it and we know django better - +Make your output pydantic models and get nicely structured GraphQL output AND nice tables of data on the page :) Overall gist ------------ @@ -157,7 +149,6 @@ What's missing from this idea - granular permissions (gotta think about nice API for this) - separate tables for different objects. -- some kind of nicer serializer Using the library ----------------- @@ -250,12 +241,9 @@ who were not as command line savvy. 2. Expose utility functions as forms for users -How output serializers might look ---------------------------------- +Customizing output +------------------ -1. One cool idea would just be to automatically convert to and from attrs classes using `cattrs` to customize output. (little more flexible in general) -2. Could just return django models that get persisted (advantage is a bit easier to see old executions) -3. Use pydantic to have some nice structure on output types :) Attrs classes ^^^^^^^^^^^^^ @@ -281,3 +269,25 @@ Better support for unmarshalling works with fast api as well https://pydantic-docs.helpmanual.io/usage/models/#data-conversion + + +Why not FastAPI? +---------------- + +This is a great point! I didn't see it before I started. +Using Django provides: + +0. FRONT END! -> key for non-technical users +1. Persistence layer is a big deal - pretty easy on-ramp to handling +2. Easy ability to add in authentication/authorization (granted FastAPI has this) +3. Literally didn't see it and we know django better + +See here for more details - https://github.com/tiangolo/fastapi + + +Why not Django Ninja? +--------------------- + +This may actually be a better option - https://github.com/vitalik/django-ninja + + diff --git a/turtle_shell/__init__.py b/turtle_shell/__init__.py index 1294d72..e1f50a3 100644 --- a/turtle_shell/__init__.py +++ b/turtle_shell/__init__.py @@ -15,14 +15,14 @@ def __init__(self): def get_registry(self): return _RegistrySingleton - def add(self, func, name=None): + def add(self, func, name=None, config=None): from .function_to_form import _Function # TODO: maybe _Function object should have overridden __new__ to keep it immutable?? :-/ name = name or func.__name__ func_obj = self.get(name) if not func_obj: - func_obj = _Function.from_function(func, name=name) + func_obj = _Function.from_function(func, name=name, config=config) self.func_name2func[func_obj.name] = func_obj else: if func_obj.func is not func: @@ -68,3 +68,5 @@ def schema(self): _RegistrySingleton = _Registry() get_registry = _Registry.get_registry + +from .function_to_form import Text diff --git a/turtle_shell/fake_pydantic_adpater.py b/turtle_shell/fake_pydantic_adpater.py new file mode 100644 index 0000000..e10c442 --- /dev/null +++ b/turtle_shell/fake_pydantic_adpater.py @@ -0,0 +1,5 @@ +def is_pydantic(func): + return False + +def maybe_use_pydantic_mutation(func_object): + return None diff --git a/turtle_shell/function_to_form.py b/turtle_shell/function_to_form.py index 196edb0..15b7c9a 100644 --- a/turtle_shell/function_to_form.py +++ b/turtle_shell/function_to_form.py @@ -22,11 +22,17 @@ from . import utils +class Text(str): + """Wrapper class to be able to handle str types""" + pass + type2field_type = {int: forms.IntegerField, str: forms.CharField, bool: forms.BooleanField, - Optional[bool]: forms.NullBooleanField, + Optional[bool]: forms.NullBooleanField, Text: forms.CharField, pathlib.Path: forms.CharField, dict: forms.JSONField} +type2widget = {Text: forms.Textarea()} + @dataclass class _Function: func: callable @@ -35,10 +41,10 @@ class _Function: doc: str @classmethod - def from_function(cls, func, *, name): - form_class = function_to_form(func, name=name) + def from_function(cls, func, *, name, config=None): + form_class = function_to_form(func, name=name, config=config) return cls(func=func, name=name, form_class=form_class, - doc=form_class.__doc__) + doc=form_class.__doc__) def doc_mapping(str) -> Dict[str, str]: @@ -173,9 +179,8 @@ def param_to_field(param: Parameter, config: dict = None) -> forms.Field: See function_to_form for config definition.""" config = config or {} - all_types = dict(type2field_type) - all_types.update(config.get("types", {})) - widgets = config.get("widgets") or {} + all_types = {**type2field_type, **(config.get("types") or {})} + widgets = {**type2widget, **(config.get("widgets") or {})} field_type = None kwargs = {} kind = get_type_from_annotation(param) diff --git a/turtle_shell/graphene_adapter.py b/turtle_shell/graphene_adapter.py index 2e4ef6e..400cd06 100644 --- a/turtle_shell/graphene_adapter.py +++ b/turtle_shell/graphene_adapter.py @@ -10,6 +10,9 @@ from . import utils # PATCH IT GOOD! import turtle_shell.graphene_adapter_jsonstring +# TODO: (try/except here with pydantic) +from turtle_shell import pydantic_adapter + _seen_names: set = set() @@ -100,15 +103,23 @@ def mutate_and_get_payload(cls, root, info, **input): @classmethod def perform_mutate(cls, form, info): obj = form.save() - obj.execute() + all_results = obj.execute() obj.save() + kwargs = {"result": obj} + if hasattr(all_results, 'dict'): + for k, f in fields.items(): + if k != 'result': + kwargs[k] = all_results + # TODO: make errors show up nicely - return cls(errors=[], result=obj) + return cls(errors=[], **kwargs) # TODO: figure out if name can be customized in class mutation_name = f'{form_class.__name__}Mutation' - DefaultOperationMutation = type(mutation_name, (DjangoFormMutation,), ({"Meta": Meta, - "perform_mutate": perform_mutate, "result": graphene.Field(ExecutionResult), "__doc__": + fields = {"result": graphene.Field(ExecutionResult)} + pydantic_adapter.maybe_add_pydantic_fields(func_object, fields) + DefaultOperationMutation = type(mutation_name, (DjangoFormMutation,), ({**fields, "Meta": Meta, + "perform_mutate": perform_mutate, "__doc__": f'Mutation form for {form_class.__name__}.\n{form_class.__doc__}', "mutate_and_get_payload": mutate_and_get_payload})) return DefaultOperationMutation diff --git a/turtle_shell/models.py b/turtle_shell/models.py index 5f81a64..acb7eaf 100644 --- a/turtle_shell/models.py +++ b/turtle_shell/models.py @@ -4,6 +4,7 @@ from turtle_shell import utils import uuid import cattr +import json class CaughtException(Exception): """An exception that was caught and saved. Generally don't need to rollback transaction with @@ -19,6 +20,8 @@ class ResultJSONEncodeException(CaughtException): class ExecutionResult(models.Model): + FIELDS_TO_SHOW_IN_LIST = [("func_name", "Function"), ("created", "Created"), ("user", "User"), + ("status", "Status")] uuid = models.UUIDField(primary_key=True, unique=True, editable=False, default=uuid.uuid4) func_name = models.CharField(max_length=512, editable=False) input_json = models.JSONField(encoder=utils.EnumAwareEncoder, decoder=utils.EnumAwareDecoder) @@ -43,12 +46,15 @@ class ExecutionStatus(models.TextChoices): def execute(self): """Execute with given input, returning caught exceptions as necessary""" + from turtle_shell import pydantic_adapter + if self.status not in (self.ExecutionStatus.CREATED, self.ExecutionStatus.RUNNING): raise ValueError("Cannot run - execution state isn't complete") func = self.get_function() + original_result = None try: # TODO: redo conversion another time! - result = func(**self.input_json) + result = original_result = func(**self.input_json) except Exception as e: # TODO: catch integrity error separately self.error_json = {"type": type(e).__name__, "message": str(e)} @@ -56,6 +62,8 @@ def execute(self): self.save() raise CaughtException(f"Failed on {self.func_name} ({type(e).__name__})", e) from e try: + if hasattr(result, "json"): + result = json.loads(result.json()) if not isinstance(result, (dict, str, tuple)): result = cattr.unstructure(result) self.output_json = result @@ -74,7 +82,7 @@ def execute(self): raise ResultJSONEncodeException(msg, e) from e else: raise e - return self + return original_result def get_function(self): # TODO: figure this out @@ -90,5 +98,14 @@ def get_absolute_url(self): return reverse(f'turtle_shell:detail-{self.func_name}', kwargs={"pk": self.pk}) def __repr__(self): - return (f'<{type(self).__name__}(pk="{self.pk}", func_name="{self.func_name}",' - f' created={self.created}, modified={self.modified})') + return (f'<{type(self).__name__}({self})') + + @property + def pydantic_object(self): + from turtle_shell import pydantic_adapter + + return pydantic_adapter.get_pydantic_object(self) + + @property + def list_entry(self) -> list: + return [getattr(self, obj_name) for obj_name, _ in self.FIELDS_TO_SHOW_IN_LIST] diff --git a/turtle_shell/pydantic_adapter.py b/turtle_shell/pydantic_adapter.py new file mode 100644 index 0000000..f3eae82 --- /dev/null +++ b/turtle_shell/pydantic_adapter.py @@ -0,0 +1,78 @@ +""" +Pydantic model handling! + +This module is designed to be a conditional import with fake pydantic instead used if pydantic not +available. +""" +import typing +from graphene_pydantic import PydanticObjectType, registry +from pydantic import BaseModel +import inspect +import graphene +import logging + +logger = logging.getLogger(__name__) + +def is_pydantic(func): + ret_type = inspect.signature(func).return_annotation + if args:= typing.get_args(ret_type): + ret_type = args[0] + return inspect.isclass(ret_type) and issubclass(ret_type, BaseModel) and ret_type + + +def get_pydantic_models_in_order(model): + """Get all nested models in order for definition""" + found = [] + for field in model.__fields__.values(): + type_ = field.type_ + print(type_) + if issubclass(type_, BaseModel): + print(f"HIT {type_}") + found.extend(get_pydantic_models_in_order(type_)) + found.append(model) + seen_classes = set() + deduped = [] + for elem in found: + if elem not in seen_classes: + deduped.append(elem) + seen_classes.add(elem) + return deduped + +def get_object_type(model) -> PydanticObjectType: + """Construct object types in order, using caching etc""" + reg = registry.get_global_registry() + classes = get_pydantic_models_in_order(model) + print("FOUND {len(classes)} classes from model") + for klass in classes: + if reg.get_type_for_model(klass): + continue + + pydantic_oject = type(klass.__name__, (PydanticObjectType,), {"Meta": type("Meta", (object,), + {"model": klass})}) + print(f"CREATED: {klass.__name__}") + assert reg.get_type_for_model(klass), klass + return reg.get_type_for_model(model) + +def maybe_add_pydantic_fields(func_object, fields): + if not (pydantic_class := is_pydantic(func_object.func)): + return + print("ADDING PYDANTIC FIELDS") + obj_name = pydantic_class.__name__ + + root_object = get_object_type(pydantic_class) + fields[obj_name[0].lower() + obj_name[1:]] = graphene.Field(root_object) + print(f"Added field {obj_name}") + +def maybe_convert_pydantic_model(result): + if isinstance(result, BaseModel): + return result.dict() + return result + + +def get_pydantic_object(execution_result): + func = execution_result.get_function() + if ret_type := is_pydantic(func): + try: + return ret_type.parse_obj(execution_result.output_json) + except Exception as e: + logger.warn(f"Hit exception unparsing {type(e).__name__}{e}", stack_info=True) diff --git a/turtle_shell/templates/turtle_shell/executionresult_create.html b/turtle_shell/templates/turtle_shell/executionresult_create.html index 4619c35..b981c58 100644 --- a/turtle_shell/templates/turtle_shell/executionresult_create.html +++ b/turtle_shell/templates/turtle_shell/executionresult_create.html @@ -3,6 +3,7 @@ {% block content %}

    New Execution for {{func_name}}

    +{{form.__doc__}} {% crispy form %} {% endblock content %} diff --git a/turtle_shell/templates/turtle_shell/executionresult_detail.html b/turtle_shell/templates/turtle_shell/executionresult_detail.html index 2808ccc..47396ff 100644 --- a/turtle_shell/templates/turtle_shell/executionresult_detail.html +++ b/turtle_shell/templates/turtle_shell/executionresult_detail.html @@ -1,25 +1,33 @@ {% extends 'base.html' %} +{% load pydantic_to_table %} {% block content %} +

    Execution for {{func_name}} ({{object.pk}})

    +
    +

    State

    +

    {{object.status}}

    -

    Input

    -
    {{object.input_json|pprint|escape}}
    -{% if object.output_json %} -

    Output

    -
    {{object.output_json|pprint|escape}}
    +{% if object.pydantic_object %} +
    +

    Results

    +{{object.pydantic_object|pydantic_model_to_table}} +
    {% endif %} -{% if object.error_json %} -

    Error

    -
    {{object.error_json|pprint|escape}}
    -{% endif %} -

    User

    -{{object.user}} -

    Created

    -{{object.created}} -

    Modified

    -{{object.modified}} +
    +

    Original Data

    + + +{% include "turtle_shell/executionresult_summaryrow.html" with key="Input" data=object.input_json %} +{% include "turtle_shell/executionresult_summaryrow.html" with key="Output" data=object.output_json %} +{% include "turtle_shell/executionresult_summaryrow.html" with key="Error" data=object.error_json %} +{% load tz %} +{% get_current_timezone as TIME_ZONE %} + + + + +
    User{{object.user}}
    Created{{object.created}} ({{TIME_ZONE}})
    Modified{{object.modified}} ({{TIME_ZONE}})
    +
    {% endblock content %} - - diff --git a/turtle_shell/templates/turtle_shell/executionresult_list.html b/turtle_shell/templates/turtle_shell/executionresult_list.html index e742e19..5020880 100644 --- a/turtle_shell/templates/turtle_shell/executionresult_list.html +++ b/turtle_shell/templates/turtle_shell/executionresult_list.html @@ -3,11 +3,29 @@ {% block content %}

    Executions for {{func_name}}

    -