diff --git a/.agents/skills/baserow-registry/SKILL.md b/.agents/skills/baserow-registry/SKILL.md new file mode 100644 index 0000000000..3c55ec33c8 --- /dev/null +++ b/.agents/skills/baserow-registry/SKILL.md @@ -0,0 +1,234 @@ +--- +name: baserow-registry +description: Explain, create, update, or use Baserow registries across backend and frontend code, including Instance/Registry patterns, registration points, model-backed type registries, serializer/API URL/import-export helpers, and the registry mixins in baserow.core.registry. +--- + +# Baserow Registry + +Use this skill when explaining or changing Baserow registries, especially: + +- backend registries based on `baserow.core.registry.Instance` and `Registry` +- model-backed typed objects such as fields, views, elements, services, widgets, domains, auth providers, workflow actions, and automation nodes +- frontend registries based on `web-frontend/modules/core/registry.js` +- registry mixins and what behavior each one adds + +## First Step + +Inspect the closest existing registry before editing. Useful patterns: + +- Core backend implementation: `backend/src/baserow/core/registry.py` +- Field types: `backend/src/baserow/contrib/database/fields/registries.py` +- Builder elements: `backend/src/baserow/contrib/builder/elements/registries.py` +- Services: `backend/src/baserow/core/services/registries.py` +- Frontend registry: `web-frontend/modules/core/registry.js` +- Registrations usually live in an app `apps.py`, module `plugin.js`, or package `config.py` + +Useful searches: + +- `rg -n "class .*Registry|_registry = .*Registry|\\.register\\(" backend/src premium/backend enterprise/backend` +- `rg -n "register\\(new .*Type|registerNamespace|getOrderedList|getAll\\(" web-frontend premium/web-frontend enterprise/web-frontend` +- `rg -n "ModelRegistryMixin|CustomFieldsRegistryMixin|APIUrlsRegistryMixin|EasyImportExportMixin" backend/src` + +## Backend Registry Shape + +A simple backend registry has an `Instance` subclass, a `Registry` subclass with a stable `name`, and a singleton registry object: + +```python +from baserow.core.registry import Instance, Registry + + +class ExampleType(Instance): + type = "example" + + +class ExampleTypeRegistry(Registry[ExampleType]): + name = "example_type" + + +example_type_registry = ExampleTypeRegistry() +``` + +Baserow convention is that backend registry instance classes end with `Type`, +registry classes end with `TypeRegistry`, and singleton variables end with +`_type_registry`. Put concrete implementations in a `*_types.py` module when +they live outside `registries.py` (for example `field_types.py` or +`service_types.py`), and keep the registry singleton in `registries.py`. + +Register instantiated types at application startup: + +```python +example_type_registry.register(ExampleType()) +``` + +Core operations: + +- `registry.register(instance)` adds an instance and calls `instance.after_register()`. +- `registry.unregister(instance_or_type)` removes it and calls `before_unregister()`. +- `registry.get(type_name)` returns the instance or raises the registry's does-not-exist exception. +- `registry.get_all()` returns registered instances. +- `registry.get_types()` returns registered type strings. +- `instance.compat_type` can map an old type name to a renamed type. + +Use custom exception classes on the registry when callers need domain-specific errors: + +```python +class ExampleTypeRegistry(Registry[ExampleType]): + name = "example_type" + does_not_exist_exception_class = ExampleTypeDoesNotExist + already_registered_exception_class = ExampleTypeAlreadyRegistered +``` + +## Model-Backed Type Registries + +Use `ModelInstanceMixin` on the type and `ModelRegistryMixin` on the registry when each registered type owns a Django model subclass: + +```python +class ExampleType(ModelInstanceMixin[Example], Instance): + type = "example" + model_class = Example + + +class ExampleTypeRegistry( + ModelRegistryMixin[Example, ExampleType], + Registry[ExampleType], +): + name = "example_type" +``` + +This enables: + +- `registry.get_by_model(model_or_instance)` +- `registry.get_for_class(model_class)` +- `registry.get_model_names()` +- model polymorphism through `.specific`, `.specific_class`, and `WithRegistry` + +For model classes participating in a registry, follow existing model patterns with `WithRegistry` and implement `get_type_registry()` when needed. + +## Custom Serializers + +Use `CustomFieldsInstanceMixin` on the type and `CustomFieldsRegistryMixin` on the registry when each type contributes model fields to generated serializers. + +Type-level properties: + +- `allowed_fields`: fields accepted during create/update. +- `serializer_field_names`: fields exposed in generated serializers. +- `request_serializer_field_names`: request-specific field list. +- `serializer_field_overrides`: DRF field overrides. +- `request_serializer_field_overrides`: request-specific overrides. +- `serializer_mixins`: serializer mixins or lazy functions returning mixins. +- `request_serializer_mixins`: request-specific mixins. +- `serializer_field_extra_kwargs`: extra DRF `Meta.extra_kwargs`. +- `serializer_extra_args`: extra serializer arguments for local conventions. + +Use `PublicCustomFieldsInstanceMixin` when public APIs must expose a narrower field set than internal APIs. Pass `extra_params={"public": True}` to select public fields, overrides, and mixins. + +## Import And Export + +Use `ImportExportMixin` when export/import is custom or not model-shaped. Implement: + +- `export_serialized(instance)` +- `import_serialized(parent, serialized_values, id_mapping, ...)` + +Use `EasyImportExportMixin` for model-backed types with direct property serialization. Define: + +- `SerializedDict`: a `TypedDict` describing exported properties. +- `parent_property_name`: the parent relation to set during import. +- `id_mapping_name`: optional mapping key to populate. +- `model_class`: the model class to create. +- `sensitive_fields`: fields omitted when `exclude_sensitive_data` is enabled. + +Override `serialize_property`, `deserialize_property`, or `create_instance_from_serialized` for file handling, ID remapping, compatibility, or custom creation. + +## API URLs And Exceptions + +Use `APIUrlsInstanceMixin` on instances that contribute API routes and `APIUrlsRegistryMixin` on the registry. Include `registry.api_urls` in the owning URL module. + +Use `MapAPIExceptionsInstanceMixin` when type-specific domain exceptions should map to API errors: + +```python +class ExampleType(MapAPIExceptionsInstanceMixin, Instance): + api_exceptions_map = { + ExampleError: ERROR_EXAMPLE, + } + +with example_type.map_api_exceptions(): + ... +``` + +## Formula-Aware Types + +Use `InstanceWithFormulaMixin` or a domain-specific subclass such as builder formula mixins when a type owns formula strings that need import rewriting. Set `simple_formula_fields` for straightforward model fields, or override `formula_generator()` for formulas inside JSON fields or nested structures. + +## Frontend Registry Shape + +Frontend registry code uses `Registerable` and `Registry` from `web-frontend/modules/core/registry.js`. + +Define a registerable type: + +```javascript +import { Registerable } from '@baserow/modules/core/registry' + +export class ExampleType extends Registerable { + static getType() { + return 'example' + } + + getOrder() { + return 10 + } +} +``` + +Register it in a plugin after the namespace exists: + +```javascript +app.$registry.register('example', new ExampleType({ app })) +``` + +Common frontend operations: + +- `registerNamespace(namespace)` +- `register(namespace, object)` +- `unregister(namespace, type)` +- `get(namespace, type)` +- `getAll(namespace)` +- `getList(namespace)` +- `getOrderedList(namespace)` +- `exists(namespace, type)` + +The frontend `getType()` should match the backend `type` when both sides describe the same feature. + +## Mixin Reference + +Backend mixins in `baserow.core.registry`: + +- `ModelInstanceMixin`: attaches `model_class` to an instance and adds content-type/object helpers. Pair with `ModelRegistryMixin`. +- `ModelRegistryMixin`: finds registered types by model class or model instance. Use for polymorphic/model-backed registries. +- `CustomFieldsInstanceMixin`: lets a type define allowed fields, serializer fields, overrides, mixins, and queryset enhancement. +- `PublicCustomFieldsInstanceMixin`: extends custom fields with public/private serializer variants selected by `extra_params["public"]`. +- `CustomFieldsRegistryMixin`: delegates serializer generation to the registered type selected by a model instance. +- `APIUrlsInstanceMixin`: lets a registered type return extra Django URL patterns. +- `APIUrlsRegistryMixin`: aggregates `get_api_urls()` from all registered types. +- `MapAPIExceptionsInstanceMixin`: maps type-specific exceptions to API errors with `map_api_exceptions()`. +- `ImportExportMixin`: abstract manual import/export contract. +- `EasyImportExportMixin`: generic model import/export implementation using a `TypedDict` property list and import ID mappings. +- `InstanceWithFormulaMixin`: iterates and rewrites formula fields during import/export workflows. + +Related non-registry mixin: + +- `baserow.core.mixins.WithRegistry`: placed on model classes so model instances can resolve their registry/type with local conventions. + +Frontend registry base classes: + +- `Registerable`: base class for frontend objects that can be registered; provides `getType()`, `type`, `getOrder()`, and `$t()`. +- `Registry`: namespace-based frontend registry for `Registerable` instances. + +## Checklist For New Or Updated Registries + +1. Choose the nearest existing registry as the pattern. +2. Define or update the instance/type class and required stable `type`. +3. Add only the mixins needed by the behavior. +4. Define or update the registry class with a stable `name`. +5. Register instances during app/plugin startup. +6. Wire API URLs, serializers, import/export, formulas, or model `WithRegistry` only when the feature needs them. +7. Add focused tests around registration, lookup, serializer generation, import/export, or frontend registry behavior based on the touched surface. diff --git a/.agents/skills/baserow-registry/agents/openai.yaml b/.agents/skills/baserow-registry/agents/openai.yaml new file mode 100644 index 0000000000..caa309aa1b --- /dev/null +++ b/.agents/skills/baserow-registry/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Baserow Registry" + short_description: "Guide Baserow registry patterns and mixins" + default_prompt: "Use $baserow-registry to explain or update a Baserow registry and choose the right registry mixins." diff --git a/.agents/skills/runtime-formulas/SKILL.md b/.agents/skills/runtime-formulas/SKILL.md index 863c83b5c3..3f5851ad1f 100644 --- a/.agents/skills/runtime-formulas/SKILL.md +++ b/.agents/skills/runtime-formulas/SKILL.md @@ -319,7 +319,7 @@ Frontend areas to inspect: Useful commands: -- `just b test backend/tests/path/to/test.py` +- `just b test tests/path/to/test.py` - `just f yarn test:core web-frontend/test/unit/path/to/test.spec.js` - `just f test` diff --git a/.env.docker-dev.example b/.env.docker-dev.example index 48ec7d6ef0..bfde1a8f4b 100644 --- a/.env.docker-dev.example +++ b/.env.docker-dev.example @@ -47,3 +47,12 @@ POSTGRES_IMAGE_VERSION=14 # AWS_S3_VERIFY=off # AWS_S3_SIGNATURE_VERSION = 's3v4' # AWS_S3_ADDRESSING_STYLE = 'path' + +# For code runner +BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE=wasmtime_quickjs +## Tweak these two only for local execution +# BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE=.local/code-runtime/bin/wasmtime +# BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH=.local/code-runtime/lib/baserow/qjs.wasm +# BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS=5 +# BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES=16777216 +# BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT=1000000000 diff --git a/.env.example b/.env.example index 99c577cd5b..b1d7d82e97 100644 --- a/.env.example +++ b/.env.example @@ -70,6 +70,12 @@ DATABASE_NAME=baserow # BASEROW_AUTOMATION_WORKFLOW_HISTORY_RATE_LIMIT_CACHE_EXPIRY_SECONDS= # BASEROW_AUTOMATION_WORKFLOW_MAX_CONSECUTIVE_ERRORS= # BASEROW_AUTOMATION_WORKFLOW_TIMEOUT_HOURS= +# BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE=wasmtime_quickjs +# BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE=wasmtime +# BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH=/usr/local/lib/baserow/qjs.wasm +# BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS=5 +# BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES=16777216 +# BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT=1000000000 # BASEROW_AUTOMATION_WORKFLOW_HISTORY_MAX_DAYS= # BASEROW_AUTOMATION_WORKFLOW_HISTORY_MAX_ENTRIES= # BASEROW_AUTOMATION_WORKFLOW_HISTORY_MIN_RETENTION_DAYS= diff --git a/.env.local-dev.example b/.env.local-dev.example index 169866e3b0..3fb8eca849 100644 --- a/.env.local-dev.example +++ b/.env.local-dev.example @@ -68,3 +68,11 @@ EMAIL_PORT=1025 # ============================================================================= MEDIA_ROOT=media MEDIA_URL=http://localhost:8000/media/ + +# For local dev to customize the code runner runtime +# BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE=wasmtime_quickjs +# BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE=.local/code-runtime/bin/wasmtime +# BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH=.local/code-runtime/lib/baserow/qjs.wasm +# BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS=5 +# BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES=16777216 +# BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT=100000000 diff --git a/backend/Dockerfile b/backend/Dockerfile index 5ef20f62f8..91a3273ff5 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -26,6 +26,87 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ chmod 0755 /usr/local/bin/su-exec +# ============================================================================= +# Builder for Wasmtime (used to execute untrusted code in a separate WASI process) +# ============================================================================= +FROM debian:trixie-slim AS wasmtime-builder +ARG TARGETARCH +ARG WASMTIME_VERSION=28.0.0 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates curl xz-utils && \ + case "${TARGETARCH}" in \ + amd64) WASMTIME_ARCH="x86_64-linux" ;; \ + arm64) WASMTIME_ARCH="aarch64-linux" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + curl -fsSL \ + "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_ARCH}.tar.xz" \ + -o /tmp/wasmtime.tar.xz && \ + tar -xJf /tmp/wasmtime.tar.xz -C /tmp && \ + mv "/tmp/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_ARCH}/wasmtime" /usr/local/bin/wasmtime && \ + chmod 0755 /usr/local/bin/wasmtime + + +# ============================================================================= +# Builder for QuickJS-NG WASI runtime +# ============================================================================= +FROM debian:trixie-slim AS quickjs-wasi-builder +ARG TARGETARCH +ARG QUICKJS_NG_VERSION=0.15.0 +ARG WASI_SDK_VERSION=27 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008 +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -y --no-install-recommends ca-certificates cmake curl make tar && \ + case "${TARGETARCH}" in \ + amd64) WASI_SDK_ARCH="x86_64" ;; \ + arm64) WASI_SDK_ARCH="arm64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" && exit 1 ;; \ + esac && \ + curl -fsSL \ + "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-${WASI_SDK_ARCH}-linux.tar.gz" \ + -o /tmp/wasi-sdk.tar.gz && \ + tar -xzf /tmp/wasi-sdk.tar.gz -C /opt && \ + mv "/opt/wasi-sdk-${WASI_SDK_VERSION}.0-${WASI_SDK_ARCH}-linux" /opt/wasi-sdk && \ + curl -fsSL \ + "https://github.com/quickjs-ng/quickjs/archive/refs/tags/v${QUICKJS_NG_VERSION}.tar.gz" \ + -o /tmp/quickjs-ng.tar.gz && \ + mkdir -p /tmp/quickjs-ng && \ + tar -xzf /tmp/quickjs-ng.tar.gz -C /tmp/quickjs-ng --strip-components=1 && \ + cmake \ + -S /tmp/quickjs-ng \ + -B /tmp/quickjs-ng/build \ + -DCMAKE_TOOLCHAIN_FILE=/opt/wasi-sdk/share/cmake/wasi-sdk.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DQJS_BUILD_WERROR=OFF && \ + cmake --build /tmp/quickjs-ng/build --target qjs_exe --parallel && \ + mkdir -p /usr/local/lib/baserow && \ + if [[ -f /tmp/quickjs-ng/build/qjs.wasm ]]; then \ + cp /tmp/quickjs-ng/build/qjs.wasm /usr/local/lib/baserow/qjs.wasm; \ + else \ + cp /tmp/quickjs-ng/build/qjs /usr/local/lib/baserow/qjs.wasm; \ + fi + + +# ============================================================================= +# Local/export target for code runner artifacts +# ============================================================================= +FROM scratch AS code-runner-artifacts + +COPY --from=wasmtime-builder /usr/local/bin/wasmtime /bin/wasmtime +COPY --from=quickjs-wasi-builder /usr/local/lib/baserow/qjs.wasm /lib/baserow/qjs.wasm + + # ============================================================================= # Production base builder stage: builds runtime dependencies only # ============================================================================= @@ -201,6 +282,8 @@ RUN groupadd --system --gid $GID ${DOCKER_USER} && \ COPY --from=tool-builder /usr/local/bin/su-exec /usr/local/bin/su-exec COPY --from=tool-builder /usr/bin/tini /usr/bin/tini +COPY --from=wasmtime-builder /usr/local/bin/wasmtime /usr/local/bin/wasmtime +COPY --from=quickjs-wasi-builder /usr/local/lib/baserow/qjs.wasm /usr/local/lib/baserow/qjs.wasm # hadolint ignore=DL3022 COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/ @@ -274,6 +357,8 @@ RUN getent group $GID || groupadd --system --gid $GID ${DOCKER_USER} && \ COPY --from=tool-builder /usr/local/bin/su-exec /usr/local/bin/su-exec COPY --from=tool-builder /usr/bin/tini /usr/bin/tini COPY --from=tool-builder /usr/bin/dos2unix /usr/local/bin/dos2unix +COPY --from=wasmtime-builder /usr/local/bin/wasmtime /usr/local/bin/wasmtime +COPY --from=quickjs-wasi-builder /usr/local/lib/baserow/qjs.wasm /usr/local/lib/baserow/qjs.wasm # hadolint ignore=DL3022 COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /bin/ @@ -359,6 +444,8 @@ RUN groupadd --system --gid $GID ${DOCKER_USER} && \ COPY ./backend/docker/mime.types /etc/ COPY --from=tool-builder /usr/local/bin/su-exec /usr/local/bin/su-exec COPY --from=tool-builder /usr/bin/tini /usr/bin/tini +COPY --from=wasmtime-builder /usr/local/bin/wasmtime /usr/local/bin/wasmtime +COPY --from=quickjs-wasi-builder /usr/local/lib/baserow/qjs.wasm /usr/local/lib/baserow/qjs.wasm RUN mkdir -p /baserow/media && chown -R $UID:$GID /baserow/ @@ -387,4 +474,4 @@ CMD ["gunicorn"] FROM local AS prod -CMD ["gunicorn"] \ No newline at end of file +CMD ["gunicorn"] diff --git a/backend/justfile b/backend/justfile index 6fd6de5a89..f696f9c529 100644 --- a/backend/justfile +++ b/backend/justfile @@ -110,9 +110,11 @@ pytest_extra_args := env("PYTEST_EXTRA_ARGS", "") # Setup & Installation # ============================================================================= -# Initialize: create Python venv with uv, lock deps, and install everything +# Initialize: create Python venv with uv, lock deps, install everything, and +# prepare local code runner runtime artifacts. [group('1 - setup')] -init: _setup-env _create-venv +[doc("Initialize backend deps and local code runner runtime artifacts")] +init: _setup-env _create-venv code-runtime uv lock uv sync @echo "" @@ -153,9 +155,42 @@ _create-venv: echo "Virtual environment already exists at {{ venv_dir }}" fi +# Build local code runner runtime artifacts +[group('1 - setup')] +[doc("Build local Wasmtime + QuickJS runtime artifacts")] +code-runtime: + #!/usr/bin/env bash + set -euo pipefail + + RUNTIME_DIR="{{ repo_root }}/.local/code-runtime" + ENV_FILE="{{ repo_root }}/.env.local" + + mkdir -p "$RUNTIME_DIR" + + docker build \ + -f "{{ repo_root }}/backend/Dockerfile" \ + --target code-runner-artifacts \ + --output "type=local,dest=$RUNTIME_DIR" \ + "{{ repo_root }}" + + chmod +x "$RUNTIME_DIR/bin/wasmtime" + + upsert_env \ + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE" \ + "$RUNTIME_DIR/bin/wasmtime" + upsert_env \ + "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH" \ + "$RUNTIME_DIR/lib/baserow/qjs.wasm" + + echo "Code runner runtime installed in $RUNTIME_DIR" + echo "Add these vars to your env file to enable the code runner:" + echo "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE=$RUNTIME_DIR/bin/wasmtime" + echo "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH=$RUNTIME_DIR/lib/baserow/qjs.wasm" + # Install dependencies (includes baserow + premium + enterprise via workspace) [group('1 - setup')] -install: _create-venv +[doc("Install backend deps and local code runner runtime artifacts")] +install: _create-venv code-runtime uv sync @echo "Baserow installed successfully!" @@ -290,7 +325,7 @@ _pytest := 'PYTHONPATH="' + test_pythonpath + ':${PYTHONPATH:-}" ' + uv_run + ' test *ARGS: _check-dev #!/usr/bin/env bash set -euo pipefail - {{ _set_pythonpath }} + {{ _load_env }} {{ _pytest }} {{ ARGS }} # Run tests with coverage report diff --git a/backend/src/baserow/contrib/automation/nodes/registries.py b/backend/src/baserow/contrib/automation/nodes/registries.py index 27b6865dbc..145c06e1f9 100644 --- a/backend/src/baserow/contrib/automation/nodes/registries.py +++ b/backend/src/baserow/contrib/automation/nodes/registries.py @@ -2,6 +2,8 @@ from django.contrib.auth.models import AbstractUser +from rest_framework.exceptions import PermissionDenied + from baserow.contrib.automation.automation_dispatch_context import ( AutomationDispatchContext, ) @@ -13,6 +15,7 @@ from baserow.contrib.automation.nodes.types import AutomationNodeDict, NodePositionType from baserow.contrib.automation.workflows.models import AutomationWorkflow from baserow.core.integrations.models import Integration +from baserow.core.models import Workspace from baserow.core.registry import ( CustomFieldsRegistryMixin, EasyImportExportMixin, @@ -23,6 +26,9 @@ PublicCustomFieldsInstanceMixin, Registry, ) +from baserow.core.services.exceptions import ( + ServiceImproperlyConfiguredDispatchException, +) from baserow.core.services.handler import ServiceHandler from baserow.core.services.registries import ServiceTypeSubClass, service_type_registry from baserow.core.services.types import DispatchResult @@ -51,6 +57,17 @@ class AutomationNodeType( class SerializedDict(AutomationNodeDict): ... + def is_deactivated(self, workspace: Workspace) -> bool: + """ + Returns whether this automation node type is deactivated for the workspace. + """ + + return False + + def raise_if_deactivated(self, workspace: Workspace) -> None: + if self.is_deactivated(workspace): + raise PermissionDenied("This automation node type is deactivated.") + @property def allowed_fields(self): return super().allowed_fields + [ @@ -325,6 +342,13 @@ def dispatch( automation_node: AutomationNode, dispatch_context: AutomationDispatchContext, ) -> DispatchResult: + if self.is_deactivated( + automation_node.workflow.get_original().automation.workspace + ): + raise ServiceImproperlyConfiguredDispatchException( + "This node type is not available for this workspace." + ) + return ServiceHandler().dispatch_service( automation_node.service.specific, dispatch_context ) diff --git a/backend/src/baserow/contrib/automation/nodes/service.py b/backend/src/baserow/contrib/automation/nodes/service.py index a509b26fa7..689bc2bbc9 100644 --- a/backend/src/baserow/contrib/automation/nodes/service.py +++ b/backend/src/baserow/contrib/automation/nodes/service.py @@ -161,6 +161,8 @@ def create_node( context=workflow, ) + node_type.raise_if_deactivated(workflow.automation.workspace) + try: reference_node = ( self.handler.get_node(reference_node_id) if reference_node_id else None @@ -238,6 +240,8 @@ def update_node( context=node, ) + node_type.raise_if_deactivated(node.workflow.automation.workspace) + # Export the 'original' node values now, as `prepare_values` # will be changing the service first, and then `update_node` # will be change the node itself. @@ -334,6 +338,8 @@ def duplicate_node( source_node.get_type().before_create(workflow, source_node, "south", "") + source_node.get_type().raise_if_deactivated(workflow.automation.workspace) + duplicated_node = self.handler.duplicate_node(source_node) workflow.get_graph().insert(duplicated_node, source_node, "south", "") @@ -383,6 +389,7 @@ def replace_node( if not existing_node: new_node_type = automation_node_type_registry.get(new_node_type_str) + new_node_type.raise_if_deactivated(automation.workspace) node_type.before_replace(node_to_replace, new_node_type) prepared_values = new_node_type.prepare_values({}, user) diff --git a/backend/src/baserow/contrib/builder/models.py b/backend/src/baserow/contrib/builder/models.py index dfdc1ebbe0..3a907b4193 100644 --- a/backend/src/baserow/contrib/builder/models.py +++ b/backend/src/baserow/contrib/builder/models.py @@ -68,6 +68,10 @@ def shared_page(self): return PageHandler().get_shared_page(self) + @property + def is_published(self) -> bool: + return hasattr(self, "published_from") + def get_workspace(self): from baserow.contrib.builder.domains.handler import DomainHandler diff --git a/backend/src/baserow/contrib/builder/workflow_actions/registries.py b/backend/src/baserow/contrib/builder/workflow_actions/registries.py index 633809bbb7..a8a14abaac 100644 --- a/backend/src/baserow/contrib/builder/workflow_actions/registries.py +++ b/backend/src/baserow/contrib/builder/workflow_actions/registries.py @@ -4,9 +4,12 @@ from django.contrib.auth.models import AbstractUser from django.core.files.storage import Storage +from rest_framework.exceptions import PermissionDenied + from baserow.contrib.builder.formula_importer import import_formula from baserow.contrib.builder.mixins import BuilderInstanceWithFormulaMixin from baserow.contrib.builder.workflow_actions.models import BuilderWorkflowAction +from baserow.core.models import Workspace from baserow.core.registry import ( CustomFieldsRegistryMixin, ModelRegistryMixin, @@ -32,6 +35,17 @@ class BuilderWorkflowActionType( parent_property_name = "page" id_mapping_name = "builder_workflow_actions" + def is_deactivated(self, workspace: Workspace) -> bool: + """ + Returns whether this workflow action type is deactivated for the workspace. + """ + + return False + + def raise_if_deactivated(self, workspace: Workspace) -> None: + if self.is_deactivated(workspace): + raise PermissionDenied("This workflow action type is deactivated.") + def prepare_values( self, values: Dict[str, Any], diff --git a/backend/src/baserow/contrib/builder/workflow_actions/service.py b/backend/src/baserow/contrib/builder/workflow_actions/service.py index e6a1b0b5e6..a09d1ce82c 100644 --- a/backend/src/baserow/contrib/builder/workflow_actions/service.py +++ b/backend/src/baserow/contrib/builder/workflow_actions/service.py @@ -39,6 +39,7 @@ from baserow.contrib.builder.workflow_actions.workflow_action_types import ( BuilderWorkflowActionType, ) +from baserow.core.exceptions import PermissionException from baserow.core.handler import CoreHandler from baserow.core.services.types import DispatchResult @@ -150,6 +151,8 @@ def create_workflow_action( context=page, ) + workflow_action_type.raise_if_deactivated(page.builder.workspace) + prepared_values = workflow_action_type.prepare_values(kwargs, user) new_workflow_action = self.handler.create_workflow_action( @@ -195,6 +198,10 @@ def update_workflow_action( else: workflow_action_type = workflow_action.get_type() + workflow_action_type.raise_if_deactivated( + workflow_action.page.builder.workspace + ) + if has_type_changed: # When a workflow action's type changes, due our polymorphism, we need # to delete the existing action and create a new one of the new type. @@ -321,10 +328,33 @@ def dispatch_action( context=workflow_action, ) - result = self.handler.dispatch_workflow_action( - workflow_action, dispatch_context + workflow_action.get_type().raise_if_deactivated( + workflow_action.page.builder.workspace ) + update_sample_data = self._can_update_sample_data( + user, workflow_action, dispatch_context + ) + if update_sample_data: + dispatch_context.use_sample_data = True + dispatch_context.update_sample_data_for = [workflow_action.service.specific] + + try: + result = self.handler.dispatch_workflow_action( + workflow_action, dispatch_context + ) + except Exception: + if update_sample_data: + workflow_action_updated.send( + self, workflow_action=workflow_action, user=user + ) + raise + + if update_sample_data: + workflow_action_updated.send( + self, workflow_action=workflow_action, user=user + ) + # Remove unfiltered fields allowed_field_names = dispatch_context.public_allowed_properties.get( "external", {} @@ -338,3 +368,34 @@ def dispatch_action( data=data, status=result.status, ) + + def _can_update_sample_data( + self, + user, + workflow_action: BuilderWorkflowServiceAction, + dispatch_context: BuilderDispatchContext, + ) -> bool: + """ + Returns whether this dispatch may persist the service sample data. + + Only authenticated dispatches against draft builders with workflow-action + update permission are allowed to refresh sample data. Published builder + copies must not mutate their service sample data. + """ + + builder = workflow_action.page.builder + + if builder.is_published: + return False + + try: + CoreHandler().check_permissions( + user, + UpdateBuilderWorkflowActionOperationType.type, + workspace=workflow_action.page.builder.workspace, + context=workflow_action, + ) + except PermissionException: + return False + + return True diff --git a/backend/src/baserow/contrib/database/airtable/handler.py b/backend/src/baserow/contrib/database/airtable/handler.py index 3e5c229825..22896f7f0a 100644 --- a/backend/src/baserow/contrib/database/airtable/handler.py +++ b/backend/src/baserow/contrib/database/airtable/handler.py @@ -150,9 +150,11 @@ def download_airtable_file( except (TypeError, ValueError): file_size_bytes = None - # If we cannot determine the size from headers, treat this as a download failure + # Fall back to measuring the actual downloaded content. This handles + # responses with Transfer-Encoding: chunked and Content-Encoding: gzip + # where no Content-Length header is present. if file_size_bytes is None: - raise FileDownloadFailed(f"Could not determine the size of file {name}.") + file_size_bytes = len(response.content) # Prevent upload to Baserow failures by excluding oversized files max_size_bytes = settings.BASEROW_FILE_UPLOAD_SIZE_LIMIT_MB @@ -192,14 +194,17 @@ def open(self, name): if name not in self.files_to_download: raise KeyError(f"File '{name}' not found in files_to_download") - response = download_airtable_file( - name=name, - download_file=self.files_to_download[name], - init_data=self.init_data, - request_id=self.request_id, - cookies=self.cookies, - headers=BASE_HEADERS, - ) + try: + response = download_airtable_file( + name=name, + download_file=self.files_to_download[name], + init_data=self.init_data, + request_id=self.request_id, + cookies=self.cookies, + headers=BASE_HEADERS, + ) + except FileDownloadFailed: + raise KeyError(f"File '{name}' could not be downloaded") stream = BytesIO(response.content) try: diff --git a/backend/src/baserow/core/code_runner/__init__.py b/backend/src/baserow/core/code_runner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/baserow/core/code_runner/exceptions.py b/backend/src/baserow/core/code_runner/exceptions.py new file mode 100644 index 0000000000..2ef3f94b7a --- /dev/null +++ b/backend/src/baserow/core/code_runner/exceptions.py @@ -0,0 +1,21 @@ +from baserow.core.exceptions import InstanceTypeDoesNotExist + + +class CodeRunnerException(Exception): + """Base exception for code runner failures.""" + + +class CodeRunnerTypeDoesNotExist(InstanceTypeDoesNotExist): + """Raised when the requested code runner type does not exist.""" + + +class CodeRunnerImproperlyConfigured(CodeRunnerException): + """Raised when a code runner is missing required configuration.""" + + +class CodeRunnerExecutionError(CodeRunnerException): + """Raised when the underlying code runtime fails.""" + + +class CodeRunnerResultError(CodeRunnerException): + """Raised when the executed code returns an unsupported result.""" diff --git a/backend/src/baserow/core/code_runner/registries.py b/backend/src/baserow/core/code_runner/registries.py new file mode 100644 index 0000000000..3d975ddd61 --- /dev/null +++ b/backend/src/baserow/core/code_runner/registries.py @@ -0,0 +1,47 @@ +from abc import ABC, abstractmethod +from typing import Any + +from django.conf import settings + +from baserow.core.code_runner.exceptions import ( + CodeRunnerImproperlyConfigured, + CodeRunnerTypeDoesNotExist, +) +from baserow.core.registry import Instance, Registry + + +class CodeRunnerType(Instance, ABC): + @abstractmethod + def run(self, context_data: dict[str, Any], code: str) -> dict[str, Any]: + """ + Execute the provided code with the given context data. + + The JavaScript code must define a `main(context)` function and return a plain + object. + """ + + +class CodeRunnerTypeRegistry(Registry[CodeRunnerType]): + name = "code_runner" + does_not_exist_exception_class = CodeRunnerTypeDoesNotExist + + +code_runner_type_registry: CodeRunnerTypeRegistry = CodeRunnerTypeRegistry() + + +def get_code_runner(code_runner_type: str | None = None) -> CodeRunnerType: + code_runner_type = code_runner_type or getattr( + settings, "ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE", "" + ) + + if not code_runner_type: + raise CodeRunnerImproperlyConfigured( + "The default code runner type is not configured." + ) + + try: + return code_runner_type_registry.get(code_runner_type) + except CodeRunnerTypeDoesNotExist as exc: + raise CodeRunnerImproperlyConfigured( + f"The code runner {code_runner_type} is not registered." + ) from exc diff --git a/backend/src/baserow/core/formula/validator.py b/backend/src/baserow/core/formula/validator.py index 42c9e431cd..56545ba77a 100644 --- a/backend/src/baserow/core/formula/validator.py +++ b/backend/src/baserow/core/formula/validator.py @@ -6,6 +6,7 @@ from typing import Any, List, Optional, Union from django.core.exceptions import ValidationError +from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import validate_email from baserow.contrib.database.fields.constants import ( @@ -291,3 +292,30 @@ def ensure_object(value: Any) -> Optional[dict]: raise ValidationError("Value is not a JSON.") from exc raise ValidationError("Value cannot be converted to a dict.") + + +class BaserowFormulaJSONEncoder(DjangoJSONEncoder): + def default(self, obj): + if isinstance(obj, timedelta): + return ensure_integer(obj) + + return super().default(obj) + + +def ensure_json(value: Any) -> Any: + """ + Ensures that the value can be converted to a JSON value. + Python types supported by Django's JSON encoder, like dates and datetimes, are + converted to their JSON representation. + + :param value: The value to ensure as a JSON value. + :return: The JSON-compatible value. + :raises ValidationError: If the value cannot be converted to JSON. + """ + + try: + return json.loads( + json.dumps(value, cls=BaserowFormulaJSONEncoder, allow_nan=False) + ) + except (TypeError, ValueError) as exc: + raise ValidationError("Value cannot be converted to a JSON value.") from exc diff --git a/backend/src/baserow/test_utils/pytest_conftest.py b/backend/src/baserow/test_utils/pytest_conftest.py index 20fc0720f0..a51c04b24e 100755 --- a/backend/src/baserow/test_utils/pytest_conftest.py +++ b/backend/src/baserow/test_utils/pytest_conftest.py @@ -372,6 +372,39 @@ def mutable_builder_workflow_action_registry(): builder_workflow_action_type_registry.registry = before +@pytest.fixture() +def mutable_automation_node_type_registry(): + from baserow.contrib.automation.nodes.registries import ( + automation_node_type_registry, + ) + + before = automation_node_type_registry.registry.copy() + automation_node_type_registry.get_for_class.cache_clear() + yield automation_node_type_registry + automation_node_type_registry.get_for_class.cache_clear() + automation_node_type_registry.registry = before + + +@pytest.fixture() +def mutable_service_type_registry(): + from baserow.core.services.registries import service_type_registry + + before = service_type_registry.registry.copy() + service_type_registry.get_for_class.cache_clear() + yield service_type_registry + service_type_registry.get_for_class.cache_clear() + service_type_registry.registry = before + + +@pytest.fixture() +def mutable_code_runner_type_registry(): + from baserow.core.code_runner.registries import code_runner_type_registry + + before = code_runner_type_registry.registry.copy() + yield code_runner_type_registry + code_runner_type_registry.registry = before + + @pytest.fixture() def mutable_job_type_registry(): with patch.object(job_type_registry, "registry", {}): diff --git a/backend/tests/baserow/contrib/builder/api/workflow_actions/test_workflow_actions_views.py b/backend/tests/baserow/contrib/builder/api/workflow_actions/test_workflow_actions_views.py index 5bc17a5ad9..303e561457 100644 --- a/backend/tests/baserow/contrib/builder/api/workflow_actions/test_workflow_actions_views.py +++ b/backend/tests/baserow/contrib/builder/api/workflow_actions/test_workflow_actions_views.py @@ -641,6 +641,67 @@ def test_dispatch_local_baserow_create_row_workflow_action(api_client, data_fixt assert response_json[color_field.name] == "Brown" assert animal_field.name not in response_json + service.refresh_from_db() + + assert service.sample_data["data"][color_field.name] == "Brown" + assert animal_field.name not in service.sample_data["data"] + assert service.sample_data["status"] == HTTP_200_OK + + +@pytest.mark.django_db +def test_dispatch_published_workflow_action_does_not_update_sample_data( + api_client, data_fixture +): + user, token = data_fixture.create_user_and_token() + table, fields, rows = data_fixture.build_table( + user=user, + columns=[ + ("Animal", "text"), + ("Color", "text"), + ], + rows=[], + ) + color_field = table.field_set.get(name="Color") + animal_field = table.field_set.get(name="Animal") + builder = data_fixture.create_builder_application(user=user) + published_builder = data_fixture.create_builder_application(workspace=None) + data_fixture.create_builder_custom_domain( + builder=builder, published_to=published_builder + ) + page = data_fixture.create_builder_page(user=user, builder=published_builder) + element = data_fixture.create_builder_button_element(page=page) + workflow_action = data_fixture.create_local_baserow_create_row_workflow_action( + page=page, element=element, event=EventTypes.CLICK, user=user + ) + service = workflow_action.service.specific + service.table = table + service.field_mappings.create(field=color_field, value="'Brown'") + service.field_mappings.create(field=animal_field, value="'Horse'") + service.save() + + url = reverse( + "api:builder:workflow_action:dispatch", + kwargs={"workflow_action_id": workflow_action.id}, + ) + + with patch( + "baserow.contrib.builder.handler.get_builder_used_property_names" + ) as used_properties_mock: + used_properties_mock.return_value = { + "all": {workflow_action.service.id: ["id", color_field.db_column]}, + "external": {workflow_action.service.id: ["id", color_field.db_column]}, + } + response = api_client.post( + url, + {}, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_200_OK + service.refresh_from_db() + assert service.sample_data is None + @pytest.mark.django_db def test_dispatch_local_baserow_create_row_workflow_action_field_constraint( diff --git a/backend/tests/baserow/contrib/builder/test_builder_application_type.py b/backend/tests/baserow/contrib/builder/test_builder_application_type.py index ad9bd4c585..c57d91f2ab 100644 --- a/backend/tests/baserow/contrib/builder/test_builder_application_type.py +++ b/backend/tests/baserow/contrib/builder/test_builder_application_type.py @@ -1437,6 +1437,19 @@ def test_builder_application_imports_login_page(data_fixture): assert new_builder.login_page.name == "foo_login_page" +@pytest.mark.django_db +def test_builder_is_published(data_fixture): + builder = data_fixture.create_builder_application() + published_builder = data_fixture.create_builder_application(workspace=None) + + data_fixture.create_builder_custom_domain( + builder=builder, published_to=published_builder + ) + + assert builder.is_published is False + assert published_builder.is_published is True + + @pytest.mark.django_db def test_delete_builder_application_with_published_builder(data_fixture): builder = data_fixture.create_builder_application() diff --git a/backend/tests/baserow/contrib/builder/workflow_actions/test_workflow_action_types.py b/backend/tests/baserow/contrib/builder/workflow_actions/test_workflow_action_types.py index 8c4d06aca9..7b86eb085d 100644 --- a/backend/tests/baserow/contrib/builder/workflow_actions/test_workflow_action_types.py +++ b/backend/tests/baserow/contrib/builder/workflow_actions/test_workflow_action_types.py @@ -20,7 +20,6 @@ from baserow.core.formula.types import BASEROW_FORMULA_MODE_SIMPLE from baserow.core.services.exceptions import InvalidServiceTypeDispatchSource from baserow.core.utils import MirrorDict -from baserow.core.workflow_actions.registries import WorkflowActionType def local_baserow_service_backed_workflow_actions(): @@ -38,71 +37,6 @@ def local_baserow_service_backed_workflow_actions(): ] -def pytest_generate_tests(metafunc): - if "workflow_action_type" in metafunc.fixturenames: - metafunc.parametrize( - "workflow_action_type", - [ - pytest.param(e, id=e.type) - for e in builder_workflow_action_type_registry.get_all() - ], - ) - - -@pytest.mark.django_db -def test_export_workflow_action(data_fixture, workflow_action_type: WorkflowActionType): - page = data_fixture.create_builder_page() - pytest_params = workflow_action_type.get_pytest_params(data_fixture) - workflow_action = data_fixture.create_workflow_action( - workflow_action_type.model_class, page=page, **pytest_params - ) - - exported = workflow_action_type.export_serialized(workflow_action) - - assert exported["id"] == workflow_action.id - assert exported["type"] == workflow_action_type.type - - serialized_pytest_params = workflow_action_type.get_pytest_params_serialized( - pytest_params - ) - for key, value in serialized_pytest_params.items(): - assert exported[key] == value - - -@pytest.mark.django_db -def test_import_workflow_action(data_fixture, workflow_action_type: WorkflowActionType): - page = data_fixture.create_builder_page() - pytest_params = workflow_action_type.get_pytest_params(data_fixture) - - page_after_import = data_fixture.create_builder_page() - element = data_fixture.create_builder_button_element(page=page_after_import) - - serialized = { - "id": 9999, - "type": workflow_action_type.type, - "page_id": 41, - "element_id": 42, - "order": 0, - "event": EventTypes.CLICK, - } - serialized.update(workflow_action_type.get_pytest_params_serialized(pytest_params)) - - id_mapping = defaultdict(MirrorDict) - id_mapping["builder_pages"] = {41: page_after_import.id} - id_mapping["builder_page_elements"] = {42: element.id} - - workflow_action = workflow_action_type.import_serialized( - page, serialized, id_mapping - ) - - assert workflow_action.id != 9999 - assert isinstance(workflow_action, workflow_action_type.model_class) - - for key, value in pytest_params.items(): - if key != "service": - assert getattr(workflow_action, key) == value - - @pytest.mark.django_db def test_export_import_upsert_row_workflow_action_type(data_fixture): user, token = data_fixture.create_user_and_token() diff --git a/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py b/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py index 55c14387aa..9f49d1b874 100644 --- a/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py +++ b/backend/tests/baserow/contrib/database/airtable/test_airtable_handler.py @@ -11,16 +11,21 @@ from rest_framework import serializers from baserow.contrib.database.airtable.config import AirtableImportConfig +from baserow.contrib.database.airtable.constants import ( + AIRTABLE_DOWNLOAD_FILE_TYPE_FETCH, +) from baserow.contrib.database.airtable.exceptions import ( AirtableBaseRequiresAuthentication, AirtableShareIsNotABase, + FileDownloadFailed, ) from baserow.contrib.database.airtable.handler import ( AirtableFileImport, AirtableHandler, + download_airtable_file, ) from baserow.contrib.database.airtable.job_types import AirtableImportJobType -from baserow.contrib.database.airtable.models import AirtableImportJob +from baserow.contrib.database.airtable.models import AirtableImportJob, DownloadFile from baserow.contrib.database.fields.models import TextField from baserow.core.exceptions import UserNotInWorkspace from baserow.core.jobs.constants import JOB_PENDING @@ -29,6 +34,14 @@ from baserow.core.user_files.models import UserFile from baserow.core.utils import Progress +STUB_AIRTABLE_FETCH_DOWNLOAD_FILE = DownloadFile( + url="https://example.com/file.pdf", + row_id="", + column_id="", + attachment_id="", + type=AIRTABLE_DOWNLOAD_FILE_TYPE_FETCH, +) + @pytest.mark.django_db @responses.activate @@ -1427,3 +1440,141 @@ def test_get_airtable_import_job(data_fixture): job = JobHandler.get_job(user, job_1.id, job_model=AirtableImportJob) assert isinstance(job, AirtableImportJob) assert job.id == job_1.id + + +@responses.activate +def test_download_airtable_file_chunked_no_content_length(): + file_content = b"%PDF-1.4 fake pdf content" * 1000 + responses.add( + responses.GET, + STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, + body=file_content, + status=200, + headers={"Transfer-Encoding": "chunked"}, + ) + + response = download_airtable_file( + name="test.pdf", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + assert response.content == file_content + + +@responses.activate +def test_download_airtable_file_with_content_length(): + file_content = b"some file bytes" + responses.add( + responses.GET, + STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, + body=file_content, + status=200, + headers={"Content-Length": str(len(file_content))}, + ) + + response = download_airtable_file( + name="file.txt", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + assert response.content == file_content + + +@responses.activate +def test_download_airtable_file_partial_content_range(): + responses.add( + responses.GET, + STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, + body=b"012345", + status=206, + headers={"Content-Range": "bytes 0-5/500000"}, + ) + + response = download_airtable_file( + name="file.pdf", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + assert response.status_code == 206 + + +@responses.activate +def test_download_airtable_file_exceeds_size_limit(): + file_content = b"x" * 100 + responses.add( + responses.GET, + STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, + body=file_content, + status=200, + headers={"Content-Length": str(len(file_content))}, + ) + + with patch("baserow.contrib.database.airtable.handler.settings") as mock_settings: + mock_settings.BASEROW_FILE_UPLOAD_SIZE_LIMIT_MB = 50 + with pytest.raises(FileDownloadFailed, match="exceeds the size limit"): + download_airtable_file( + name="big.bin", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + + +@responses.activate +def test_download_airtable_file_chunked_exceeds_size_limit(): + file_content = b"x" * 100 + responses.add( + responses.GET, + STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, + body=file_content, + status=200, + headers={"Transfer-Encoding": "chunked"}, + ) + + with patch("baserow.contrib.database.airtable.handler.settings") as mock_settings: + mock_settings.BASEROW_FILE_UPLOAD_SIZE_LIMIT_MB = 50 + with pytest.raises(FileDownloadFailed, match="exceeds the size limit"): + download_airtable_file( + name="big.bin", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + + +@responses.activate +def test_download_airtable_file_http_error(): + responses.add(responses.GET, STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, status=404) + + with pytest.raises(FileDownloadFailed, match="HTTP 404"): + download_airtable_file( + name="missing.pdf", + download_file=STUB_AIRTABLE_FETCH_DOWNLOAD_FILE, + init_data={}, + request_id="req1", + cookies={}, + ) + + +@responses.activate +def test_airtable_file_import_open_download_failure_raises_key_error(): + responses.add(responses.GET, STUB_AIRTABLE_FETCH_DOWNLOAD_FILE.url, status=500) + + file_import = AirtableFileImport( + init_data={}, + request_id="req1", + cookies={}, + ) + file_import.add_files({"broken.pdf": STUB_AIRTABLE_FETCH_DOWNLOAD_FILE}) + + with pytest.raises(KeyError, match="could not be downloaded"): + with file_import.open("broken.pdf"): + pass diff --git a/backend/tests/baserow/contrib/integrations/core/test_service_types.py b/backend/tests/baserow/contrib/integrations/core/test_service_types.py deleted file mode 100644 index 9193857ff3..0000000000 --- a/backend/tests/baserow/contrib/integrations/core/test_service_types.py +++ /dev/null @@ -1,22 +0,0 @@ -from baserow.contrib.integrations.core.service_types import ( - CoreHTTPRequestServiceType, - CorePeriodicServiceType, - CoreRouterServiceType, - CoreServiceType, - CoreSMTPEmailServiceType, -) -from baserow.core.services.registries import DispatchTypes, service_type_registry - - -def test_core_service_type_dispatch_types(): - core_dispatch_types = { - service_type.type: service_type.dispatch_types - for service_type in service_type_registry.get_all() - if isinstance(service_type, CoreServiceType) - } - assert core_dispatch_types == { - CoreHTTPRequestServiceType.type: [DispatchTypes.ACTION], - CoreSMTPEmailServiceType.type: [DispatchTypes.ACTION], - CoreRouterServiceType.type: [DispatchTypes.ACTION], - CorePeriodicServiceType.type: [DispatchTypes.EVENT], - } diff --git a/backend/tests/baserow/core/formula/test_validator.py b/backend/tests/baserow/core/formula/test_validator.py index 867896c5a4..17a530a4fc 100644 --- a/backend/tests/baserow/core/formula/test_validator.py +++ b/backend/tests/baserow/core/formula/test_validator.py @@ -9,6 +9,7 @@ ensure_datetime, ensure_duration, ensure_integer, + ensure_json, ensure_string, ) @@ -75,6 +76,29 @@ def test_ensure_datetime_throws_exception_for_invalid_value(value): assert exc.value.args[0] == "Value cannot be converted to a datetime." +def test_ensure_json_returns_json_compatible_values(): + assert ensure_json( + { + "datetime": datetime(2024, 12, 17, 12, 0, 0), + "date": date(2024, 12, 17), + "duration": timedelta(days=1, hours=2, minutes=3, seconds=4), + "array": [1, True, None], + } + ) == { + "datetime": "2024-12-17T12:00:00", + "date": "2024-12-17", + "duration": 93784, + "array": [1, True, None], + } + + +@pytest.mark.parametrize("value", [float("nan"), object()]) +def test_ensure_json_throws_exception_for_invalid_value(value): + with pytest.raises(ValidationError) as exc: + ensure_json(value) + assert exc.value.args[0] == "Value cannot be converted to a JSON value." + + @pytest.mark.parametrize( "value,expected", [ diff --git a/changelog/entries/unreleased/bug/5440_fix_for_builder_and_lower_roles_should_not_be_able_to_import.json b/changelog/entries/unreleased/bug/5440_fix_for_builder_and_lower_roles_should_not_be_able_to_import.json new file mode 100644 index 0000000000..3101ba2766 --- /dev/null +++ b/changelog/entries/unreleased/bug/5440_fix_for_builder_and_lower_roles_should_not_be_able_to_import.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Hide workspace import/export options for Builder and lower roles.", + "issue_origin": "github", + "issue_number": 5440, + "domain": "core", + "bullet_points": [], + "created_at": "2026-06-02" +} diff --git a/changelog/entries/unreleased/bug/5472_fix_airtable_import_when_cdn_omits_contentlength.json b/changelog/entries/unreleased/bug/5472_fix_airtable_import_when_cdn_omits_contentlength.json new file mode 100644 index 0000000000..161de3bdce --- /dev/null +++ b/changelog/entries/unreleased/bug/5472_fix_airtable_import_when_cdn_omits_contentlength.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fix airtable import when CDN omits Content-Length", + "issue_origin": "github", + "issue_number": 5472, + "domain": "database", + "bullet_points": [], + "created_at": "2026-06-08" +} \ No newline at end of file diff --git a/changelog/entries/unreleased/bug/fix_multiple_select_no_value.json b/changelog/entries/unreleased/bug/fix_multiple_select_no_value.json new file mode 100644 index 0000000000..136707ca97 --- /dev/null +++ b/changelog/entries/unreleased/bug/fix_multiple_select_no_value.json @@ -0,0 +1,9 @@ +{ + "type": "bug", + "message": "Fix multiple select field no value in row edit modal.", + "issue_origin": "github", + "issue_number": null, + "domain": "builder", + "bullet_points": [], + "created_at": "2026-06-08" +} diff --git a/changelog/entries/unreleased/feature/5424_add_code_runner_service_to_execute_arbitrary_javascript_code.json b/changelog/entries/unreleased/feature/5424_add_code_runner_service_to_execute_arbitrary_javascript_code.json new file mode 100644 index 0000000000..a45ec0a1cf --- /dev/null +++ b/changelog/entries/unreleased/feature/5424_add_code_runner_service_to_execute_arbitrary_javascript_code.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Add code service to execute arbitrary JavaScript code in workflow/builder", + "issue_origin": "github", + "issue_number": 5424, + "domain": "integration", + "bullet_points": [], + "created_at": "2026-06-02" +} diff --git a/changelog/entries/unreleased/feature/allow_to_test_actions_in_preview_mode_to_capture_the_output.json b/changelog/entries/unreleased/feature/allow_to_test_actions_in_preview_mode_to_capture_the_output.json new file mode 100644 index 0000000000..c475d68fa9 --- /dev/null +++ b/changelog/entries/unreleased/feature/allow_to_test_actions_in_preview_mode_to_capture_the_output.json @@ -0,0 +1,9 @@ +{ + "type": "feature", + "message": "Allow to test actions in preview mode to capture the output", + "issue_origin": "github", + "issue_number": null, + "domain": "builder", + "bullet_points": [], + "created_at": "2026-06-02" +} \ No newline at end of file diff --git a/changelog/src/handler.py b/changelog/src/handler.py index fc17f13602..de1c5eb589 100644 --- a/changelog/src/handler.py +++ b/changelog/src/handler.py @@ -66,6 +66,7 @@ def add_entry( bullet_points=bullet_points, ) json.dump(entry, entry_file, indent=4) + entry_file.write(LINE_BREAK_CHARACTER) return full_path diff --git a/deploy/all-in-one/Dockerfile b/deploy/all-in-one/Dockerfile index 6193b1f185..ef7235451c 100644 --- a/deploy/all-in-one/Dockerfile +++ b/deploy/all-in-one/Dockerfile @@ -64,6 +64,8 @@ RUN ln -s /opt/yarn-v1.22.22/bin/yarn /usr/local/bin/yarn # Copy su-exec from backend image (already built with pinned version + SHA256 verification) COPY --from=backend_image /usr/local/bin/su-exec /usr/local/bin/su-exec +COPY --from=backend_image /usr/local/bin/wasmtime /usr/local/bin/wasmtime +COPY --from=backend_image /usr/local/lib/baserow/qjs.wasm /usr/local/lib/baserow/qjs.wasm ENV POSTGRES_LOCATION=/etc/postgresql/15/main \ UV_PROJECT_ENVIRONMENT=/baserow/venv \ @@ -151,4 +153,4 @@ COPY --chown=$UID:$GID deploy/plugins/*.sh /baserow/plugins/ HEALTHCHECK --interval=60s CMD ["/bin/bash", "/baserow/backend/docker/docker-entrypoint.sh", "backend-healthcheck"] ENTRYPOINT ["/baserow.sh"] CMD ["start"] -EXPOSE 80 \ No newline at end of file +EXPOSE 80 diff --git a/docker-compose.no-caddy.yml b/docker-compose.no-caddy.yml index f235566e0a..fdd9c077ec 100644 --- a/docker-compose.no-caddy.yml +++ b/docker-compose.no-caddy.yml @@ -200,6 +200,12 @@ x-backend-variables: BASEROW_DEADLOCK_MAX_RETRIES: BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_SERIES: BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS: + BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE: + BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE: + BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH: + BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS: + BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES: + BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT: BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL: BASEROW_ENTERPRISE_ASSISTANT_LLM_TEMPERATURE: BASEROW_EMBEDDINGS_API_URL: @@ -268,6 +274,7 @@ services: BASEROW_AUTOMATION_WORKFLOW_HISTORY_MAX_ENTRIES: BASEROW_AUTOMATION_WORKFLOW_HISTORY_MIN_RETENTION_DAYS: BASEROW_AUTOMATION_WORKFLOW_HISTORY_CLEANUP_INTERVAL_MINUTES: + BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE: depends_on: - backend networks: diff --git a/docker-compose.yml b/docker-compose.yml index 1ee0859859..12ddc7e755 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -252,6 +252,12 @@ x-backend-variables: BASEROW_DEADLOCK_MAX_RETRIES: BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_SERIES: BASEROW_PREMIUM_GROUPED_AGGREGATE_SERVICE_MAX_AGG_BUCKETS: + BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE: + BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE: + BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH: + BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS: + BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES: + BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT: BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL: BASEROW_ENTERPRISE_ASSISTANT_LLM_TEMPERATURE: BASEROW_EMBEDDINGS_API_URL: @@ -353,6 +359,7 @@ services: BASEROW_AUTOMATION_WORKFLOW_HISTORY_MIN_RETENTION_DAYS: BASEROW_AUTOMATION_WORKFLOW_HISTORY_CLEANUP_INTERVAL_MINUTES: BASEROW_INTEGRATIONS_PERIODIC_MINUTE_MIN: + BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE: BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL: depends_on: - backend diff --git a/docs/development/justfile.md b/docs/development/justfile.md index 90f3420ad1..21b80e7db7 100644 --- a/docs/development/justfile.md +++ b/docs/development/justfile.md @@ -45,6 +45,8 @@ curl -LsSf https://astral.sh/uv/install.sh | sh ```bash just init # Install backend + frontend dependencies, create .env.local + # Also builds local code runner artifacts +just b code-runtime # Rebuild local Wasmtime + QuickJS runtime artifacts just pre-commit-install # Install Git pre-commit hooks locally ``` diff --git a/docs/installation/configuration.md b/docs/installation/configuration.md index 841e9f066d..636696a529 100644 --- a/docs/installation/configuration.md +++ b/docs/installation/configuration.md @@ -67,6 +67,12 @@ The installation methods referred to in the variable descriptions are: | BASEROW\_PG\_FULLTEXT\_SEARCH\_UPDATE\_DATA\_THROTTLE\_SECONDS | The delay before triggering the task that updates full-text search data. A higher value reduces how often the task runs when many changes occur but delays how soon newly modified values become searchable. Only one update task will run per table. The value is in seconds | 2 (seconds) | | BASEROW\_ASGI\_HTTP\_MAX\_CONCURRENCY | Specifies a limit for concurrent requests handled by a single gunicorn worker. The default is: no limit. | | | BASEROW\_IMPORT\_EXPORT\_RESOURCE\_REMOVAL\_AFTER\_DAYS | Specifies the number of days after which an import/export resource will be automatically deleted. | 5 (days) | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_DEFAULT\_TYPE | The default enterprise code execution type. Set to an empty value to disable enterprise code execution, the code builder workflow action, and the code automation node. | wasmtime\_quickjs | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_WASMTIME\_EXECUTABLE | The path or executable name used to run Wasmtime for enterprise code execution. | wasmtime | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_QUICKJS\_WASM\_PATH | The path to the QuickJS WASM runtime used by enterprise code execution. | /usr/local/lib/baserow/qjs.wasm | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_TIMEOUT\_SECONDS | The maximum number of seconds enterprise code execution can run before timing out. | 5 | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_MEMORY\_LIMIT\_BYTES | The maximum number of bytes the enterprise code WebAssembly linear memory can grow to. | 67108864 | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_FUEL\_LIMIT | The maximum amount of Wasmtime execution fuel available to each enterprise code execution. Set to `0` to disable fuel limiting. | 1000000000 | ### Rate Limiting @@ -232,6 +238,17 @@ Baserow can throttle the number of concurrent requests a single user (or, option | BASEROW\_AUTOMATION\_WORKFLOW\_HISTORY\_MAX\_DAYS | The number of days automation workflow history entries are retained. | 30 | | BASEROW\_AUTOMATION\_WORKFLOW\_HISTORY\_MAX\_ENTRIES | The maximum number of workflow history entries retained per workflow. | 50 | +### Code runner Configuration + +| Name | Description | Defaults | +|------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------| +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_DEFAULT\_TYPE | The default enterprise code runner type used to execute custom code services. Set to an empty value to disable enterprise code runner registration. | wasmtime_quickjs | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_WASMTIME\_EXECUTABLE | The wasmtime executable used by the `wasmtime_quickjs` code runner. | wasmtime | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_QUICKJS\_WASM\_PATH | The path to the QuickJS WASM runtime used by the `wasmtime_quickjs` code runner. | /usr/local/lib/baserow/qjs.wasm | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_TIMEOUT\_SECONDS | The maximum number of seconds a code runner process can execute before it times out. | 5 | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_MEMORY\_LIMIT\_BYTES | The maximum memory size in bytes available to a code runner process. | 16777216 | +| BASEROW\_ENTERPRISE\_CODE\_RUNNER\_FUEL\_LIMIT | The wasmtime fuel limit used to cap code runner instruction execution. Set to 0 to disable the fuel limit. | 1000000000 | + ### Backend Application Builder Configuration | Name | Description | Defaults | |---------------------------|--------------------------------------------------------------------------------------------------------------------------|------------------------| diff --git a/enterprise/backend/src/baserow_enterprise/apps.py b/enterprise/backend/src/baserow_enterprise/apps.py index 0d5d53a783..8cc114cba7 100755 --- a/enterprise/backend/src/baserow_enterprise/apps.py +++ b/enterprise/backend/src/baserow_enterprise/apps.py @@ -5,6 +5,33 @@ from tqdm import tqdm +def register_code_runner_features(): + if not getattr(settings, "ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE", ""): + return + + from baserow.contrib.automation.nodes.registries import ( + automation_node_type_registry, + ) + from baserow.contrib.builder.workflow_actions.registries import ( + builder_workflow_action_type_registry, + ) + from baserow.core.code_runner.registries import code_runner_type_registry + from baserow.core.services.registries import service_type_registry + from baserow_enterprise.automation.nodes.node_types import CoreCodeNodeType + from baserow_enterprise.builder.workflow_actions.workflow_action_types import ( + CoreCodeActionType, + ) + from baserow_enterprise.code_runner.code_runner_types import ( + WasmtimeQuickJSCodeRunnerType, + ) + from baserow_enterprise.integrations.core.service_types import CoreCodeServiceType + + code_runner_type_registry.register(WasmtimeQuickJSCodeRunnerType()) + service_type_registry.register(CoreCodeServiceType()) + builder_workflow_action_type_registry.register(CoreCodeActionType()) + automation_node_type_registry.register(CoreCodeNodeType()) + + class BaserowEnterpriseConfig(AppConfig): name = "baserow_enterprise" @@ -226,6 +253,8 @@ def ready(self): app_auth_provider_type_registry.register(SamlAppAuthProviderType()) app_auth_provider_type_registry.register(OpenIdConnectAppAuthProviderType()) + register_code_runner_features() + from baserow.contrib.builder.elements.registries import element_type_registry from baserow_enterprise.builder.elements.element_types import ( AuthFormElementType, diff --git a/enterprise/backend/src/baserow_enterprise/automation/__init__.py b/enterprise/backend/src/baserow_enterprise/automation/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/automation/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/src/baserow_enterprise/automation/nodes/__init__.py b/enterprise/backend/src/baserow_enterprise/automation/nodes/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/automation/nodes/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/src/baserow_enterprise/automation/nodes/models.py b/enterprise/backend/src/baserow_enterprise/automation/nodes/models.py new file mode 100644 index 0000000000..cb6cacccca --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/automation/nodes/models.py @@ -0,0 +1,4 @@ +from baserow.contrib.automation.nodes.models import AutomationActionNode + + +class CoreCodeActionNode(AutomationActionNode): ... diff --git a/enterprise/backend/src/baserow_enterprise/automation/nodes/node_types.py b/enterprise/backend/src/baserow_enterprise/automation/nodes/node_types.py new file mode 100644 index 0000000000..5cd36cb1a5 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/automation/nodes/node_types.py @@ -0,0 +1,17 @@ +from baserow.contrib.automation.nodes.node_types import AutomationNodeActionNodeType +from baserow_enterprise.automation.nodes.models import CoreCodeActionNode +from baserow_enterprise.features import CODE_RUNNER +from baserow_enterprise.integrations.core.service_types import CoreCodeServiceType +from baserow_premium.license.handler import LicenseHandler + + +class CoreCodeNodeType(AutomationNodeActionNodeType): + type = "code" + model_class = CoreCodeActionNode + service_type = CoreCodeServiceType.type + + def is_deactivated(self, workspace) -> bool: + return not LicenseHandler.workspace_has_feature(CODE_RUNNER, workspace) + + def raise_if_deactivated(self, workspace) -> None: + LicenseHandler.raise_if_workspace_doesnt_have_feature(CODE_RUNNER, workspace) diff --git a/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/__init__.py b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/models.py b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/models.py new file mode 100644 index 0000000000..fc75072896 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/models.py @@ -0,0 +1,6 @@ +from baserow.contrib.builder.workflow_actions.models import ( + BuilderWorkflowServiceAction, +) + + +class CoreCodeWorkflowAction(BuilderWorkflowServiceAction): ... diff --git a/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/workflow_action_types.py b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/workflow_action_types.py new file mode 100644 index 0000000000..9a9520dcb4 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/builder/workflow_actions/workflow_action_types.py @@ -0,0 +1,25 @@ +from typing import Dict + +from baserow.contrib.builder.workflow_actions.workflow_action_types import ( + BuilderWorkflowServiceActionType, +) +from baserow_enterprise.builder.workflow_actions.models import CoreCodeWorkflowAction +from baserow_enterprise.features import CODE_RUNNER +from baserow_enterprise.integrations.core.service_types import CoreCodeServiceType +from baserow_premium.license.handler import LicenseHandler + + +class CoreCodeActionType(BuilderWorkflowServiceActionType): + type = "code" + model_class = CoreCodeWorkflowAction + service_type = CoreCodeServiceType.type + + def get_pytest_params(self, pytest_data_fixture) -> Dict[str, int]: + service = pytest_data_fixture.create_enterprise_core_code_service() + return {"service": service} + + def is_deactivated(self, workspace) -> bool: + return not LicenseHandler.workspace_has_feature(CODE_RUNNER, workspace) + + def raise_if_deactivated(self, workspace) -> None: + LicenseHandler.raise_if_workspace_doesnt_have_feature(CODE_RUNNER, workspace) diff --git a/enterprise/backend/src/baserow_enterprise/code_runner/__init__.py b/enterprise/backend/src/baserow_enterprise/code_runner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/enterprise/backend/src/baserow_enterprise/code_runner/code_runner_types.py b/enterprise/backend/src/baserow_enterprise/code_runner/code_runner_types.py new file mode 100644 index 0000000000..e15805abe7 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/code_runner/code_runner_types.py @@ -0,0 +1,370 @@ +import json +import os +import selectors +import subprocess # nosec +import time +from typing import Any + +from django.conf import settings + +from baserow.core.code_runner.exceptions import ( + CodeRunnerExecutionError, + CodeRunnerImproperlyConfigured, + CodeRunnerResultError, +) +from baserow.core.code_runner.registries import ( + CodeRunnerType, +) + +_MSGPACK_MIN_INTEGER = -(2**63) +_MSGPACK_MAX_INTEGER = 2**64 - 1 + + +class WasmtimeQuickJSCodeRunnerType(CodeRunnerType): + """ + Runs user JavaScript in a QuickJS WASI module launched by wasmtime. + """ + + type = "wasmtime_quickjs" + output_size_limit_bytes = 1024 * 1024 + fuel_exhausted_error_message = "The code instruction limit was reached." + memory_or_stack_limit_error_message = ( + "The code exceeded the runtime memory or stack limit." + ) + wasm_trap_error_message = "The code stopped because of a runtime execution error." + + def __init__( + self, + wasmtime_executable: str | None = None, + quickjs_wasm_path: str | None = None, + timeout_seconds: int | None = None, + memory_limit_bytes: int | None = None, + fuel_limit: int | None = None, + output_size_limit_bytes: int | None = None, + ): + self.wasmtime_executable = wasmtime_executable or getattr( + settings, + "ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE", + "wasmtime", + ) + self.quickjs_wasm_path = quickjs_wasm_path or getattr( + settings, + "ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH", + "", + ) + self.timeout_seconds = timeout_seconds or getattr( + settings, + "ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS", + 5, + ) + self.memory_limit_bytes = memory_limit_bytes or getattr( + settings, + "ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES", + 16 * 1024 * 1024, + ) + self.fuel_limit = ( + fuel_limit + if fuel_limit is not None + else getattr(settings, "ENTERPRISE_CODE_RUNNER_FUEL_LIMIT", 1_000_000_000) + ) + self.output_size_limit_bytes = ( + output_size_limit_bytes or self.output_size_limit_bytes + ) + + def run(self, context_data: dict[str, Any], code: str) -> dict[str, Any]: + if not self.quickjs_wasm_path: + raise CodeRunnerImproperlyConfigured( + "The QuickJS WASM runtime path is not configured." + ) + + completed_process = self._run_process(context_data, code) + + try: + payload = json.loads(completed_process.stdout) + except json.JSONDecodeError as exc: + raise CodeRunnerExecutionError( + "The code returned an invalid response." + ) from exc + + if "error" in payload: + raise CodeRunnerExecutionError(f"Code execution failed: {payload['error']}") + + result = payload.get("result") + if not isinstance(result, dict): + raise CodeRunnerResultError("The code must return an object.") + + self._validate_result(result) + + return result + + def _validate_result(self, value: Any) -> None: + """ + Validate that the returned JSON value can be sent through Channels Redis. + + Channels Redis serializes websocket payloads with msgpack. Python can + parse JSON integers with arbitrary precision, but msgpack can only + encode integers in the 64-bit signed/unsigned range. + """ + + if isinstance(value, bool): + return + + if isinstance(value, int): + if value < _MSGPACK_MIN_INTEGER or value > _MSGPACK_MAX_INTEGER: + raise CodeRunnerResultError( + "The code returned an integer outside the supported range." + ) + return + + if isinstance(value, dict): + for child_value in value.values(): + self._validate_result(child_value) + return + + if isinstance(value, list): + for child_value in value: + self._validate_result(child_value) + + def _run_process( + self, context_data: dict[str, Any], code: str + ) -> subprocess.CompletedProcess: + # Using wastime with no host access at all for best security + command = [ + self.wasmtime_executable, + "run", + "-W", + f"timeout={self.timeout_seconds}s", # Limits the execution time + "-W", + f"max-memory-size={self.memory_limit_bytes}", # Limits memory consumption + "-W", + "trap-on-grow-failure=true", # Ensure a proper exception on memory limit + ] + # Fuel limit allow a precise number of instruction to be executed + # Extra security in addition to memory and timeout. + if self.fuel_limit > 0: + command.extend(["-W", f"fuel={self.fuel_limit}"]) + + command.extend( + [ + self.quickjs_wasm_path, + "--std", # Give access to STD but it's removed in JS code + "--eval", + self._runner_source(), + ] + ) + # We send the context data and the code through STDIN to the js process + payload = json.dumps({"context": context_data, "code": code}) + + try: + completed_process = self._communicate_with_output_limit(command, payload) + except subprocess.TimeoutExpired as exc: + raise CodeRunnerExecutionError("The code timed out.") from exc + except subprocess.CalledProcessError as exc: + message = exc.stderr.strip() or exc.stdout.strip() or str(exc) + raise CodeRunnerExecutionError( + self._format_process_error_message(message) + ) from exc + except OSError as exc: + raise CodeRunnerExecutionError( + self._format_process_error_message(str(exc)) + ) from exc + + if completed_process.returncode != 0: + message = ( + completed_process.stderr.strip() + or completed_process.stdout.strip() + or f"Command returned non-zero exit status {completed_process.returncode}." + ) + raise CodeRunnerExecutionError(self._format_process_error_message(message)) + + return completed_process + + def _format_process_error_message(self, message: str) -> str: + if self._is_fuel_exhausted_error(message): + return self.fuel_exhausted_error_message + + if self._is_memory_or_stack_limit_error(message): + return self.memory_or_stack_limit_error_message + + if self._is_wasm_trap_error(message): + return self.wasm_trap_error_message + + return self._sanitize_process_error_message(message) + + def _is_fuel_exhausted_error(self, message: str) -> bool: + return "all fuel consumed" in message.lower() + + def _is_memory_or_stack_limit_error(self, message: str) -> bool: + lower_message = message.lower() + return ( + "memory fault at wasm address" in lower_message + or "out of bounds memory access" in lower_message + ) + + def _is_wasm_trap_error(self, message: str) -> bool: + return "wasm trap:" in message.lower() + + def _sanitize_process_error_message(self, message: str) -> str: + """ + Remove configured host paths from runtime errors before exposing them. + + Wasmtime can include the executable and WASM module paths in trap + messages, for example when fuel is exhausted. Those paths are host + implementation details and must not be visible to code-runner callers. + """ + + for path in (self.wasmtime_executable, self.quickjs_wasm_path): + if path and os.path.sep in path: + message = message.replace(path, os.path.basename(path)) + return message + + def _communicate_with_output_limit( + self, command: list[str], payload: str + ) -> subprocess.CompletedProcess: + """ + Run the code runner command with the given stdin payload. + + This intentionally avoids subprocess.communicate() so stdout and stderr can + be read incrementally by _read_bounded_process_output(), enforcing both the + execution timeout and per-stream output size limit while the process is + still running. If setup, writing, reading, or waiting fails, the child + process is killed before the original exception is re-raised. + """ + + process = subprocess.Popen( # noqa: S603 + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + if process.stdin is None: + raise CodeRunnerExecutionError("The code stdin pipe was not created.") + process.stdin.write(payload.encode()) + process.stdin.close() + + stdout, stderr = self._read_bounded_process_output(process) + return subprocess.CompletedProcess( + command, + process.wait(timeout=0), + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), + ) + except Exception: + if process.poll() is None: + process.kill() + process.wait() + raise + + def _read_bounded_process_output( + self, process: subprocess.Popen + ) -> tuple[bytes, bytes]: + """ + Read stdout and stderr from a running process without blocking indefinitely. + + Both streams are switched to nonblocking mode and monitored together so a + full stderr pipe cannot block stdout, or vice versa. Reading continues + until both streams close, the configured timeout expires, or either stream + exceeds the configured output size limit. + """ + + if process.stdout is None or process.stderr is None: + raise CodeRunnerExecutionError("The code output pipes were not created.") + + selector = selectors.DefaultSelector() + streams = { + process.stdout: bytearray(), + process.stderr: bytearray(), + } + + for stream in streams: + os.set_blocking(stream.fileno(), False) + selector.register(stream, selectors.EVENT_READ) + + deadline = time.monotonic() + self.timeout_seconds + + while selector.get_map(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise subprocess.TimeoutExpired(process.args, self.timeout_seconds) + + for key, _ in selector.select(timeout=remaining): + stream = key.fileobj + chunk = stream.read(8192) + if not chunk: + selector.unregister(stream) + continue + + output = streams[stream] + output.extend(chunk) + if len(output) > self.output_size_limit_bytes: + raise CodeRunnerExecutionError("The code produced too much output.") + + return bytes(streams[process.stdout]), bytes(streams[process.stderr]) + + def _runner_source(self) -> str: + """ + Build the JavaScript wrapper that QuickJS evaluates before running + user-submitted code. + + Security model + -------------- + The real isolation boundary is Wasmtime + WASI: the guest is launched + with no --dir, no --env, and no inherit-network/env capabilities, so + user code has no host I/O surface even with arbitrary JS execution + inside the guest. Do not weaken those flags on the assumption that the + scrubbing in this wrapper provides containment — it does not. + + Defense in depth (this wrapper) + ------------------------------- + qjs --std injects bridge globals (std, os, bjson, print, console, + scriptArgs, ...) that connect JS to host capabilities. We capture the + I/O we need into closure-local references and then ``delete`` those + globals so user code cannot reach them, even after recovering a real + ``Function`` via tricks like ``(function(){}).constructor``. This is + safe because this QuickJS build does NOT register std/os/bjson as + importable modules either (dynamic ``import("std")`` raises + "could not load module"), so deletion from globalThis is sufficient at + the JS layer. + + We intentionally do NOT shadow ECMAScript intrinsics such as + ``Function``, ``eval``, or ``globalThis``: such shadowing is bypassed + in one line via the constructor-escape trick, and pretending otherwise + creates a false sense of security. Those intrinsics are harmless on + their own — they grant no host access without bridge globals. + """ + return """ +const _stdOutPuts = std.out.puts.bind(std.out); +try { + const _input = JSON.parse(std.in.getline()); + const _createFunction = Function; + + // Delete the qjs --std bridge globals so user code cannot reach host I/O. + // Wasmtime + WASI is the real isolation boundary; this is defense in + // depth. Keep this list in sync with what qjs --std injects. + const _HOST_GLOBALS = [ + "std", "os", "bjson", + "print", "console", + "scriptArgs", "execArgv", "argv0", + "gc", "queueMicrotask", "performance", "navigator", + "atob", "btoa", + ]; + for (const _name of _HOST_GLOBALS) { + delete globalThis[_name]; + } + + const run = _createFunction("context", ` + "use strict"; + ${_input.code} + if (typeof main !== "function") { + throw new Error("The code must define a main function."); + } + return main(context); + `); + const result = run(_input.context); + _stdOutPuts(JSON.stringify({ result }) + "\\n"); +} catch (error) { + _stdOutPuts(JSON.stringify({ error: String(error && error.message || error) }) + "\\n"); +} +""".strip() diff --git a/enterprise/backend/src/baserow_enterprise/config/settings/settings.py b/enterprise/backend/src/baserow_enterprise/config/settings/settings.py index 338d73e23b..89b64b67a6 100644 --- a/enterprise/backend/src/baserow_enterprise/config/settings/settings.py +++ b/enterprise/backend/src/baserow_enterprise/config/settings/settings.py @@ -75,6 +75,27 @@ def setup(settings): or 4 ) + settings.ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE = os.getenv( + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE", "wasmtime" + ) + settings.ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE = os.getenv( + "BASEROW_ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE", "wasmtime_quickjs" + ) + settings.ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH = os.getenv( + "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH", + "/usr/local/lib/baserow/qjs.wasm", + ) + settings.ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS = int( + os.getenv("BASEROW_ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS", "") or 5 + ) + settings.ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES = int( + os.getenv("BASEROW_ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES", "") + or 16 * 1024 * 1024 + ) + settings.ENTERPRISE_CODE_RUNNER_FUEL_LIMIT = int( + os.getenv("BASEROW_ENTERPRISE_CODE_RUNNER_FUEL_LIMIT", "") or 1_000_000_000 + ) + # AI Assistant settings settings.BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL = os.getenv( "BASEROW_ENTERPRISE_ASSISTANT_LLM_MODEL", "" diff --git a/enterprise/backend/src/baserow_enterprise/features.py b/enterprise/backend/src/baserow_enterprise/features.py index fef1e0f013..88438ab6da 100644 --- a/enterprise/backend/src/baserow_enterprise/features.py +++ b/enterprise/backend/src/baserow_enterprise/features.py @@ -14,6 +14,7 @@ BUILDER_NO_BRANDING = "application_no_branding" BUILDER_FILE_INPUT = "builder_file_input" BUILDER_CUSTOM_CODE = "builder_custom_code" +CODE_RUNNER = "code_runner" DATE_DEPENDENCY = "date_dependency" DATA_SCANNER = "data_scanner" diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/__init__.py b/enterprise/backend/src/baserow_enterprise/integrations/core/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/api/__init__.py b/enterprise/backend/src/baserow_enterprise/integrations/core/api/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/api/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/api/serializers.py b/enterprise/backend/src/baserow_enterprise/integrations/core/api/serializers.py new file mode 100644 index 0000000000..0354258897 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/api/serializers.py @@ -0,0 +1,34 @@ +import re + +from rest_framework import serializers +from rest_framework.exceptions import ValidationError + +from baserow.core.formula.serializers import FormulaSerializerField +from baserow_enterprise.integrations.core.models import CoreCodeServiceInjection + + +def validate_injection_name(value): + valid_name_regex = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") + + if not valid_name_regex.match(value): + raise ValidationError( + "The name must be a valid variable name containing only alphanumeric " + "characters or underscores, and must not start with a number." + ) + + return value + + +class CoreCodeServiceInjectionSerializer(serializers.ModelSerializer): + """ + Serializer for code service injections. + """ + + name = serializers.CharField( + allow_blank=False, max_length=255, validators=[validate_injection_name] + ) + formula = FormulaSerializerField() + + class Meta: + model = CoreCodeServiceInjection + fields = ["id", "name", "formula"] diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/models.py b/enterprise/backend/src/baserow_enterprise/integrations/core/models.py new file mode 100644 index 0000000000..1cad0d688c --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/models.py @@ -0,0 +1,38 @@ +from django.core.validators import MaxLengthValidator +from django.db import models + +from baserow.core.formula.field import FormulaField +from baserow.core.services.models import Service + +CORE_CODE_SERVICE_CODE_MAX_LENGTH = 4096 + + +class CoreCodeService(Service): + """ + A service for executing arbitrary code. + """ + + code = models.TextField( + blank=True, + help_text="The code to execute.", + max_length=CORE_CODE_SERVICE_CODE_MAX_LENGTH, + validators=[MaxLengthValidator(CORE_CODE_SERVICE_CODE_MAX_LENGTH)], + ) + + +class CoreCodeServiceInjection(models.Model): + """ + A value extracted from the dispatch context and injected into the code context. + """ + + service = models.ForeignKey( + CoreCodeService, on_delete=models.CASCADE, related_name="injections" + ) + name = models.CharField( + max_length=255, + help_text="The variable name to use in the code executor context.", + ) + formula = FormulaField( + blank=True, + help_text="The formula used to extract the variable value from the context.", + ) diff --git a/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py b/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py new file mode 100644 index 0000000000..218acdfad7 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/integrations/core/service_types.py @@ -0,0 +1,215 @@ +from typing import Any, Dict, Generator, List, Tuple + +from genson import SchemaBuilder +from rest_framework import serializers + +from baserow.contrib.integrations.core.service_types import CoreServiceType +from baserow.core.code_runner.exceptions import ( + CodeRunnerExecutionError, + CodeRunnerImproperlyConfigured, + CodeRunnerResultError, +) +from baserow.core.code_runner.registries import ( + get_code_runner, +) +from baserow.core.formula.types import BaserowFormulaObject +from baserow.core.formula.validator import ensure_json +from baserow.core.registry import Instance +from baserow.core.services.dispatch_context import DispatchContext +from baserow.core.services.exceptions import ( + ServiceImproperlyConfiguredDispatchException, + UnexpectedDispatchException, +) +from baserow.core.services.models import Service +from baserow.core.services.registries import DispatchTypes +from baserow.core.services.types import DispatchResult, FormulaToResolve, ServiceDict +from baserow_enterprise.integrations.core.models import ( + CORE_CODE_SERVICE_CODE_MAX_LENGTH, + CoreCodeService, + CoreCodeServiceInjection, +) + + +class CoreCodeServiceType(CoreServiceType): + type = "code" + model_class = CoreCodeService + dispatch_types = [DispatchTypes.ACTION] + allowed_fields = ["code"] + serializer_field_names = ["code", "injections"] + request_serializer_field_names = ["code", "injections"] + + class SerializedDict(ServiceDict): + code: str + injections: List[Dict[str, str]] + + def get_schema_name(self, service: CoreCodeService) -> str: + return f"Code{service.id}Schema" + + def generate_schema( + self, + service: CoreCodeService, + allowed_fields: List[str] | None = None, + ) -> Dict[str, Any] | None: + if service.sample_data is None or "data" not in service.sample_data: + return None + + schema_builder = SchemaBuilder() + schema_builder.add_object(service.sample_data["data"]) + + return { + **schema_builder.to_schema(), + "title": self.get_schema_name(service), + } + + @property + def serializer_field_overrides(self): + from baserow_enterprise.integrations.core.api.serializers import ( + CoreCodeServiceInjectionSerializer, + ) + + return { + "code": serializers.CharField( + help_text=CoreCodeService._meta.get_field("code").help_text, + allow_blank=True, + max_length=CORE_CODE_SERVICE_CODE_MAX_LENGTH, + required=False, + ), + "injections": CoreCodeServiceInjectionSerializer( + many=True, + required=False, + help_text="The values to inject into the code executor context.", + ), + } + + @property + def request_serializer_field_overrides(self): + return self.serializer_field_overrides + + def after_create( + self, + instance: CoreCodeService, + values: Dict, + ): + if "injections" in values: + instance.injections.all().delete() + CoreCodeServiceInjection.objects.bulk_create( + [ + CoreCodeServiceInjection( + service=instance, + name=injection["name"], + formula=injection["formula"], + ) + for injection in values["injections"] + ] + ) + + def after_update( + self, + instance: CoreCodeService, + values: Dict, + changes: Dict[str, Tuple], + ): + return self.after_create(instance, values) + + def formulas_to_resolve(self, service: CoreCodeService) -> list[FormulaToResolve]: + return [ + FormulaToResolve( + injection.name, + injection.formula, + ensure_json, + f'injection "{injection.name}"', + ) + for injection in service.injections.all() + ] + + def extract_properties( + self, service: Service, path: List[str], **kwargs + ) -> List[str]: + if path: + return [path[0]] + return [] + + def dispatch_data( + self, + service: CoreCodeService, + resolved_values: Dict[str, Any], + dispatch_context: DispatchContext, + ) -> Dict[str, Any]: + try: + return get_code_runner().run(resolved_values, service.code) + except CodeRunnerImproperlyConfigured as exc: + raise ServiceImproperlyConfiguredDispatchException(str(exc)) from exc + except CodeRunnerResultError as exc: + raise ServiceImproperlyConfiguredDispatchException(str(exc)) from exc + except CodeRunnerExecutionError as exc: + raise UnexpectedDispatchException(str(exc)) from exc + + def dispatch_transform(self, data: Dict[str, Any]) -> DispatchResult: + return DispatchResult(data=data) + + def formula_generator( + self, service: Service + ) -> Generator[str | Instance, str, None]: + yield from super().formula_generator(service) + + for injection in service.injections.all(): + new_formula = yield BaserowFormulaObject.to_formula(injection.formula) + if new_formula is not None: + injection.formula = new_formula + yield injection + + def serialize_property( + self, + service: CoreCodeService, + prop_name: str, + files_zip=None, + storage=None, + cache=None, + ): + if prop_name == "injections": + return [ + { + "name": injection.name, + "formula": injection.formula, + } + for injection in service.injections.all() + ] + + return super().serialize_property( + service, prop_name, files_zip=files_zip, storage=storage, cache=cache + ) + + def create_instance_from_serialized( + self, + serialized_values, + id_mapping, + files_zip=None, + storage=None, + cache=None, + **kwargs, + ): + injections = serialized_values.pop("injections", []) + + service = super().create_instance_from_serialized( + serialized_values, + id_mapping, + files_zip=files_zip, + storage=storage, + cache=cache, + **kwargs, + ) + + CoreCodeServiceInjection.objects.bulk_create( + [ + CoreCodeServiceInjection( + **injection, + service=service, + ) + for injection in injections + ] + ) + + return service + + def enhance_queryset(self, queryset): + return super().enhance_queryset(queryset).prefetch_related("injections") diff --git a/enterprise/backend/src/baserow_enterprise/license_types.py b/enterprise/backend/src/baserow_enterprise/license_types.py index 8bd750c5bf..004f400640 100755 --- a/enterprise/backend/src/baserow_enterprise/license_types.py +++ b/enterprise/backend/src/baserow_enterprise/license_types.py @@ -8,6 +8,7 @@ BUILDER_FILE_INPUT, BUILDER_NO_BRANDING, BUILDER_SSO, + CODE_RUNNER, DATA_SCANNER, DATA_SYNC, DATE_DEPENDENCY, @@ -104,6 +105,7 @@ class EnterpriseWithoutSupportLicenseType(AdvancedLicenseType): ENTERPRISE_SETTINGS, SECURE_FILE_SERVE, DATA_SCANNER, + CODE_RUNNER, ] def handle_seat_overflow(self, seats_taken: int, license_object: License): diff --git a/enterprise/backend/src/baserow_enterprise/migrations/0061_core_code_service_action_node.py b/enterprise/backend/src/baserow_enterprise/migrations/0061_core_code_service_action_node.py new file mode 100644 index 0000000000..fd875f0c99 --- /dev/null +++ b/enterprise/backend/src/baserow_enterprise/migrations/0061_core_code_service_action_node.py @@ -0,0 +1,130 @@ +import django.core.validators +import django.db.models.deletion +from django.db import migrations, models + +import baserow.core.formula.field + + +class Migration(migrations.Migration): + dependencies = [ + ("automation", "0029_alter_automationworkflowhistory_original_workflow"), + ("baserow_enterprise", "0060_datascan_whole_words_datascanresult_cell_value"), + ("builder", "0068_alter_page_unique_together"), + ("core", "0107_twofactorauthprovidermodel_totpauthprovidermodel_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="CoreCodeActionNode", + fields=[ + ( + "automationnode_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="automation.automationnode", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("automation.automationnode",), + ), + migrations.CreateModel( + name="CoreCodeService", + fields=[ + ( + "service_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="core.service", + ), + ), + ( + "code", + models.TextField( + blank=True, + help_text="The code to execute.", + max_length=4096, + validators=[django.core.validators.MaxLengthValidator(4096)], + ), + ), + ], + options={ + "abstract": False, + }, + bases=("core.service",), + ), + migrations.CreateModel( + name="CoreCodeServiceInjection", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField( + help_text="The variable name to use in the code executor context.", + max_length=255, + ), + ), + ( + "formula", + baserow.core.formula.field.FormulaField( + blank=True, + help_text="The formula used to extract the variable value from the context.", + ), + ), + ( + "service", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="injections", + to="baserow_enterprise.corecodeservice", + ), + ), + ], + ), + migrations.CreateModel( + name="CoreCodeWorkflowAction", + fields=[ + ( + "builderworkflowaction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="builder.builderworkflowaction", + ), + ), + ( + "service", + models.ForeignKey( + help_text="The service which this action is associated with.", + on_delete=django.db.models.deletion.CASCADE, + to="core.service", + ), + ), + ], + options={ + "abstract": False, + }, + bases=("builder.builderworkflowaction",), + ), + ] diff --git a/enterprise/backend/src/baserow_enterprise/models.py b/enterprise/backend/src/baserow_enterprise/models.py index 7508268be1..95cf0a0903 100644 --- a/enterprise/backend/src/baserow_enterprise/models.py +++ b/enterprise/backend/src/baserow_enterprise/models.py @@ -1,13 +1,19 @@ +from baserow_enterprise.automation.nodes.models import CoreCodeActionNode from baserow_enterprise.builder.custom_code.models import ( BuilderCustomCode, BuilderCustomScript, ) from baserow_enterprise.builder.elements.models import AuthFormElement +from baserow_enterprise.builder.workflow_actions.models import CoreCodeWorkflowAction from baserow_enterprise.data_sync.models import LocalBaserowTableDataSync from baserow_enterprise.date_dependency.models import DateDependency from baserow_enterprise.integrations.common.sso.saml.models import ( SamlAppAuthProviderModel, ) +from baserow_enterprise.integrations.core.models import ( + CoreCodeService, + CoreCodeServiceInjection, +) from baserow_enterprise.integrations.models import ( LocalBaserowPasswordAppAuthProvider, LocalBaserowUserSource, @@ -28,4 +34,8 @@ "BuilderCustomScript", "BuilderCustomCode", "DateDependency", + "CoreCodeService", + "CoreCodeServiceInjection", + "CoreCodeWorkflowAction", + "CoreCodeActionNode", ] diff --git a/enterprise/backend/tests/baserow_enterprise_tests/builder/workflow_actions/test_ent_workflow_action_types.py b/enterprise/backend/tests/baserow_enterprise_tests/builder/workflow_actions/test_ent_workflow_action_types.py new file mode 100644 index 0000000000..2e36d163e4 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/builder/workflow_actions/test_ent_workflow_action_types.py @@ -0,0 +1,81 @@ +from collections import defaultdict + +import pytest + +from baserow.contrib.builder.workflow_actions.models import EventTypes +from baserow.contrib.builder.workflow_actions.registries import ( + builder_workflow_action_type_registry, +) +from baserow.core.utils import MirrorDict +from baserow.core.workflow_actions.registries import WorkflowActionType + + +def pytest_generate_tests(metafunc): + if "workflow_action_type" in metafunc.fixturenames: + metafunc.parametrize( + "workflow_action_type", + [ + pytest.param(e, id=e.type) + for e in builder_workflow_action_type_registry.get_all() + ], + ) + + +@pytest.mark.django_db +def test_export_workflow_action( + enterprise_data_fixture, workflow_action_type: WorkflowActionType +): + page = enterprise_data_fixture.create_builder_page() + pytest_params = workflow_action_type.get_pytest_params(enterprise_data_fixture) + workflow_action = enterprise_data_fixture.create_workflow_action( + workflow_action_type.model_class, page=page, **pytest_params + ) + + exported = workflow_action_type.export_serialized(workflow_action) + + assert exported["id"] == workflow_action.id + assert exported["type"] == workflow_action_type.type + + serialized_pytest_params = workflow_action_type.get_pytest_params_serialized( + pytest_params + ) + for key, value in serialized_pytest_params.items(): + assert exported[key] == value + + +@pytest.mark.django_db +def test_import_workflow_action( + enterprise_data_fixture, workflow_action_type: WorkflowActionType +): + page = enterprise_data_fixture.create_builder_page() + pytest_params = workflow_action_type.get_pytest_params(enterprise_data_fixture) + + page_after_import = enterprise_data_fixture.create_builder_page() + element = enterprise_data_fixture.create_builder_button_element( + page=page_after_import + ) + + serialized = { + "id": 9999, + "type": workflow_action_type.type, + "page_id": 41, + "element_id": 42, + "order": 0, + "event": EventTypes.CLICK, + } + serialized.update(workflow_action_type.get_pytest_params_serialized(pytest_params)) + + id_mapping = defaultdict(MirrorDict) + id_mapping["builder_pages"] = {41: page_after_import.id} + id_mapping["builder_page_elements"] = {42: element.id} + + workflow_action = workflow_action_type.import_serialized( + page, serialized, id_mapping + ) + + assert workflow_action.id != 9999 + assert isinstance(workflow_action, workflow_action_type.model_class) + + for key, value in pytest_params.items(): + if key != "service": + assert getattr(workflow_action, key) == value diff --git a/enterprise/backend/tests/baserow_enterprise_tests/enterprise_fixtures.py b/enterprise/backend/tests/baserow_enterprise_tests/enterprise_fixtures.py index 865ccc28d0..45a43a892a 100644 --- a/enterprise/backend/tests/baserow_enterprise_tests/enterprise_fixtures.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/enterprise_fixtures.py @@ -2,6 +2,7 @@ from baserow.core.cache import local_cache from baserow.core.models import Settings +from baserow_enterprise.integrations.core.models import CoreCodeService from baserow_enterprise.models import Role, RoleAssignment, Team, TeamSubject from baserow_premium.license.models import License @@ -37,6 +38,9 @@ def delete_all_licenses(self): License.objects.all().delete() local_cache.clear() + def create_enterprise_core_code_service(self, **kwargs): + return self.create_service(CoreCodeService, **kwargs) + def create_team(self, **kwargs): if "name" not in kwargs: kwargs["name"] = self.fake.name() diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/__init__.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/__init__.py index e69de29bb2..8b13789179 100644 --- a/enterprise/backend/tests/baserow_enterprise_tests/integrations/__init__.py +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/__init__.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/__init__.py @@ -0,0 +1 @@ + diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runner_license.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runner_license.py new file mode 100644 index 0000000000..a7172960dc --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runner_license.py @@ -0,0 +1,153 @@ +from django.http import HttpRequest +from django.test import override_settings + +import pytest + +from baserow.contrib.automation.nodes.service import AutomationNodeService +from baserow.contrib.builder.data_sources.builder_dispatch_context import ( + BuilderDispatchContext, +) +from baserow.contrib.builder.workflow_actions.models import EventTypes +from baserow.contrib.builder.workflow_actions.service import ( + BuilderWorkflowActionService, +) +from baserow.core.services.exceptions import ( + ServiceImproperlyConfiguredDispatchException, +) +from baserow.test_utils.pytest_conftest import FakeDispatchContext +from baserow_enterprise.apps import register_code_runner_features +from baserow_enterprise.automation.nodes.node_types import CoreCodeNodeType +from baserow_enterprise.builder.workflow_actions.models import CoreCodeWorkflowAction +from baserow_enterprise.builder.workflow_actions.workflow_action_types import ( + CoreCodeActionType, +) +from baserow_premium.license.exceptions import FeaturesNotAvailableError + + +def unregister_code_runner_features( + builder_workflow_action_registry, + automation_node_type_registry, + service_type_registry, + code_runner_type_registry, +): + builder_workflow_action_registry.registry.pop("code", None) + automation_node_type_registry.registry.pop("code", None) + service_type_registry.registry.pop("code", None) + code_runner_type_registry.registry.pop("wasmtime_quickjs", None) + + +@pytest.fixture +def code_runner_registered( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, +): + unregister_code_runner_features( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, + ) + + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE="wasmtime_quickjs"): + register_code_runner_features() + yield + + +@pytest.mark.django_db +def test_core_code_workflow_action_requires_enterprise_license( + enterprise_data_fixture, code_runner_registered +): + user = enterprise_data_fixture.create_user() + page = enterprise_data_fixture.create_builder_page(user=user) + element = enterprise_data_fixture.create_builder_button_element(page=page) + workflow_action_type = CoreCodeActionType() + + enterprise_data_fixture.delete_all_licenses() + + assert workflow_action_type.is_deactivated(page.builder.workspace) + + with pytest.raises(FeaturesNotAvailableError): + BuilderWorkflowActionService().create_workflow_action( + user, + workflow_action_type, + page=page, + element=element, + event=EventTypes.CLICK, + ) + + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + assert not workflow_action_type.is_deactivated(page.builder.workspace) + + +@pytest.mark.django_db +def test_core_code_automation_node_requires_enterprise_license( + enterprise_data_fixture, code_runner_registered +): + user = enterprise_data_fixture.create_user() + workflow = enterprise_data_fixture.create_automation_workflow(user) + node_type = CoreCodeNodeType() + + enterprise_data_fixture.delete_all_licenses() + + assert node_type.is_deactivated(workflow.automation.workspace) + + with pytest.raises(FeaturesNotAvailableError): + AutomationNodeService().create_node( + user, + node_type, + workflow, + reference_node_id=workflow.get_trigger().id, + position="south", + output="", + ) + + with override_settings(DEBUG=True): + enterprise_data_fixture.enable_enterprise() + assert not node_type.is_deactivated(workflow.automation.workspace) + + +@pytest.mark.django_db +def test_core_code_workflow_action_dispatch_requires_enterprise_license( + enterprise_data_fixture, code_runner_registered +): + user = enterprise_data_fixture.create_user() + page = enterprise_data_fixture.create_builder_page(user=user) + element = enterprise_data_fixture.create_builder_button_element(page=page) + service = enterprise_data_fixture.create_enterprise_core_code_service() + workflow_action = CoreCodeWorkflowAction.objects.create( + page=page, + element=element, + event=EventTypes.CLICK, + service=service, + order=0, + ) + + enterprise_data_fixture.delete_all_licenses() + + dispatch_context = BuilderDispatchContext(HttpRequest(), page) + with pytest.raises(FeaturesNotAvailableError): + BuilderWorkflowActionService().dispatch_action( + user, workflow_action, dispatch_context + ) + + +@pytest.mark.django_db +def test_core_code_automation_node_dispatch_requires_enterprise_license( + enterprise_data_fixture, code_runner_registered +): + user = enterprise_data_fixture.create_user() + workflow = enterprise_data_fixture.create_automation_workflow(user) + service = enterprise_data_fixture.create_enterprise_core_code_service() + node = enterprise_data_fixture.create_automation_node( + workflow=workflow, + type=CoreCodeNodeType.type, + service=service, + ) + + enterprise_data_fixture.delete_all_licenses() + + with pytest.raises(ServiceImproperlyConfiguredDispatchException): + node.get_type().dispatch(node, FakeDispatchContext()) diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runners.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runners.py new file mode 100644 index 0000000000..3e27429e1a --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_code_runners.py @@ -0,0 +1,711 @@ +import os +import subprocess +import sys +from pathlib import Path +from shutil import which + +from django.test import override_settings + +import pytest + +from baserow.core.code_runner.exceptions import ( + CodeRunnerExecutionError, + CodeRunnerImproperlyConfigured, + CodeRunnerResultError, +) +from baserow.core.code_runner.registries import ( + get_code_runner, +) +from baserow_enterprise.apps import register_code_runner_features +from baserow_enterprise.code_runner.code_runner_types import ( + WasmtimeQuickJSCodeRunnerType, +) + +wasmtime_executable = os.environ.get( + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE" +) +quickjs_wasm_path = os.environ.get("BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH") +runtime_variables_are_configured = ( + wasmtime_executable + and (Path(wasmtime_executable).is_file() or which(wasmtime_executable)) + and quickjs_wasm_path + and Path(quickjs_wasm_path).is_file() +) + + +@override_settings( + ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE="wasmtime-test", + ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH="/runtime/qjs.wasm", + ENTERPRISE_CODE_RUNNER_TIMEOUT_SECONDS=7, + ENTERPRISE_CODE_RUNNER_MEMORY_LIMIT_BYTES=1024 * 1024, + ENTERPRISE_CODE_RUNNER_FUEL_LIMIT=100_000, +) +def test_wasmtime_quickjs_code_runner_runs_code_in_subprocess(monkeypatch): + calls = [] + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + calls.append((command, kwargs)) + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "payload = sys.stdin.read();" + 'assert \'"context": {"value": 2}\' in payload;' + "assert 'function main(context)' in payload;" + 'sys.stdout.write(\'{"result": {"newValue": 4}}\')' + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + result = WasmtimeQuickJSCodeRunnerType().run( + {"value": 2}, + "function main(context) { return { newValue: context.value * 2 } }", + ) + + assert result == {"newValue": 4} + command, kwargs = calls[0] + assert command[:11] == [ + "wasmtime-test", + "run", + "-W", + "timeout=7s", + "-W", + "max-memory-size=1048576", + "-W", + "trap-on-grow-failure=true", + "-W", + "fuel=100000", + "/runtime/qjs.wasm", + ] + assert command[11] == "--std" + assert command[12] == "--eval" + assert "std.in.getline()" in command[13] + assert 'createFunction("context"' in command[13] + assert "globalThis.eval(input.code)" not in command[13] + assert kwargs["stdin"] == subprocess.PIPE + assert kwargs["stdout"] == subprocess.PIPE + assert kwargs["stderr"] == subprocess.PIPE + + +def test_wasmtime_quickjs_code_runner_uses_explicit_memory_limit(monkeypatch): + calls = [] + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + calls.append((command, kwargs)) + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stdout.write('{\"result\": {}}')", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm", + memory_limit_bytes=2 * 1024 * 1024, + ) + + runner.run({}, "function main() { return { newValue: 4 } }") + + assert "max-memory-size=2097152" in calls[0][0] + + +def test_wasmtime_quickjs_code_runner_uses_explicit_fuel_limit(monkeypatch): + calls = [] + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + calls.append((command, kwargs)) + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stdout.write('{\"result\": {}}')", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm", + fuel_limit=200_000, + ) + + runner.run({}, "function main() { return { newValue: 4 } }") + + assert "fuel=200000" in calls[0][0] + + +def test_wasmtime_quickjs_code_runner_can_disable_fuel_limit(monkeypatch): + calls = [] + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + calls.append((command, kwargs)) + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stdout.write('{\"result\": {}}')", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm", + fuel_limit=0, + ) + + runner.run({}, "function main() { return { newValue: 4 } }") + + assert not any(arg.startswith("fuel=") for arg in calls[0][0]) + + +def unregister_code_runner_features( + builder_workflow_action_registry, + automation_node_type_registry, + service_type_registry, + code_runner_type_registry, +): + builder_workflow_action_registry.registry.pop("code", None) + automation_node_type_registry.registry.pop("code", None) + service_type_registry.registry.pop("code", None) + code_runner_type_registry.registry.pop("wasmtime_quickjs", None) + + +def test_wasmtime_quickjs_code_runner_type_is_registered( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, +): + unregister_code_runner_features( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, + ) + + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE="wasmtime_quickjs"): + register_code_runner_features() + + assert isinstance( + mutable_code_runner_type_registry.get("wasmtime_quickjs"), + WasmtimeQuickJSCodeRunnerType, + ) + + +def test_get_code_runner_requires_default_code_runner_type(): + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE=""): + with pytest.raises(CodeRunnerImproperlyConfigured): + get_code_runner() + + +def test_get_code_runner_uses_default_code_runner_type( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, +): + unregister_code_runner_features( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, + ) + + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE="wasmtime_quickjs"): + register_code_runner_features() + + assert isinstance(get_code_runner(), WasmtimeQuickJSCodeRunnerType) + + +def test_code_runner_features_are_not_registered_without_default_type( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, +): + unregister_code_runner_features( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, + ) + + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE=""): + register_code_runner_features() + + assert "code" not in mutable_builder_workflow_action_registry.registry + assert "code" not in mutable_automation_node_type_registry.registry + assert "code" not in mutable_service_type_registry.registry + assert "wasmtime_quickjs" not in mutable_code_runner_type_registry.registry + + +def test_wasmtime_quickjs_code_runner_requires_quickjs_wasm_path(): + with override_settings(ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH=""): + with pytest.raises(CodeRunnerImproperlyConfigured): + WasmtimeQuickJSCodeRunnerType().run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_rejects_non_object_result(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stdout.write('{\"result\": 1}')", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + with override_settings( + ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH="/runtime/qjs.wasm" + ): + with pytest.raises(CodeRunnerResultError): + WasmtimeQuickJSCodeRunnerType().run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_rejects_oversized_integer_result(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "sys.stdin.read();" + "sys.stdout.write(" + '\'{"result": {"nested": [{"value": 18446744073709551616}]}}\'' + ")" + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType(quickjs_wasm_path="/runtime/qjs.wasm") + + with pytest.raises(CodeRunnerResultError, match="outside the supported range"): + runner.run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_allows_msgpack_range_integer_result( + monkeypatch, +): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "sys.stdin.read();" + "sys.stdout.write(" + '\'{"result": {"value": 18446744073709551615, "ok": true}}\'' + ")" + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType(quickjs_wasm_path="/runtime/qjs.wasm") + + assert runner.run({}, "function main() {}") == { + "value": 18446744073709551615, + "ok": True, + } + + +def test_wasmtime_quickjs_code_runner_maps_process_errors(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stderr.write('boom'); sys.exit(1)", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + with override_settings( + ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH="/runtime/qjs.wasm" + ): + with pytest.raises(CodeRunnerExecutionError, match="boom"): + WasmtimeQuickJSCodeRunnerType().run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_formats_fuel_exhaustion_error(monkeypatch): + popen = subprocess.Popen + wasmtime_path = "/opt/baserow/runtime/bin/wasmtime" + quickjs_path = "/opt/baserow/runtime/modules/qjs.wasm" + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "sys.stdin.read();" + "sys.stderr.write(" + f"'error while executing {wasmtime_path}: " + f"failed to run main module {quickjs_path}: all fuel consumed'" + ");" + "sys.exit(1)" + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + wasmtime_executable=wasmtime_path, + quickjs_wasm_path=quickjs_path, + ) + + with pytest.raises(CodeRunnerExecutionError) as exc_info: + runner.run({}, "function main() {}") + + message = str(exc_info.value) + assert message == "The code instruction limit was reached." + assert wasmtime_path not in message + assert quickjs_path not in message + assert "all fuel consumed" not in message + + +def test_wasmtime_quickjs_code_runner_formats_memory_trap_error(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "sys.stdin.read();" + "sys.stderr.write(" + "'Error: failed to run main module `qjs.wasm`\\n\\n" + "Caused by:\\n" + " 0: failed to invoke command default\\n" + " 1: error while executing at wasm backtrace:\\n" + " 0: 0x1acfc - qjs!JS_CallInternal\\n" + " 537: 0x11c0 - qjs!_start\\n" + " 2: memory fault at wasm address 0x100000094 " + "in linear memory of size 0x50000\\n" + " 3: wasm trap: out of bounds memory access'" + ");" + "sys.exit(1)" + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType(quickjs_wasm_path="/runtime/qjs.wasm") + + with pytest.raises(CodeRunnerExecutionError) as exc_info: + runner.run({}, "function main(context) { return main() }") + + message = str(exc_info.value) + assert message == "The code exceeded the runtime memory or stack limit." + assert "wasm backtrace" not in message + assert "qjs!JS_CallInternal" not in message + + +def test_wasmtime_quickjs_code_runner_formats_generic_wasm_trap_error(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + ( + "import sys;" + "sys.stdin.read();" + "sys.stderr.write('error while executing at wasm backtrace: " + "wasm trap: unreachable');" + "sys.exit(1)" + ), + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType(quickjs_wasm_path="/runtime/qjs.wasm") + + with pytest.raises(CodeRunnerExecutionError) as exc_info: + runner.run({}, "function main() {}") + + message = str(exc_info.value) + assert message == "The code stopped because of a runtime execution error." + assert "wasm trap" not in message + + +def test_wasmtime_quickjs_code_runner_redacts_startup_error_paths(monkeypatch): + wasmtime_path = "/opt/baserow/runtime/bin/wasmtime" + + def fake_popen(command, **kwargs): + raise OSError(f"No such file or directory: '{wasmtime_path}'") + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + wasmtime_executable=wasmtime_path, + quickjs_wasm_path="/opt/baserow/runtime/modules/qjs.wasm", + ) + + with pytest.raises(CodeRunnerExecutionError) as exc_info: + runner.run({}, "function main() {}") + + message = str(exc_info.value) + assert wasmtime_path not in message + assert "wasmtime" in message + + +def test_wasmtime_quickjs_code_runner_limits_stdout(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stdout.write('x' * 32)", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm", + output_size_limit_bytes=16, + ) + + with pytest.raises(CodeRunnerExecutionError, match="too much output"): + runner.run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_limits_stderr(monkeypatch): + popen = subprocess.Popen + + def fake_popen(command, **kwargs): + return popen( + [ + sys.executable, + "-c", + "import sys; sys.stdin.read(); sys.stderr.write('x' * 32)", + ], + **kwargs, + ) + + monkeypatch.setattr(subprocess, "Popen", fake_popen) + + runner = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm", + output_size_limit_bytes=16, + ) + + with pytest.raises(CodeRunnerExecutionError, match="too much output"): + runner.run({}, "function main() {}") + + +def test_wasmtime_quickjs_code_runner_uses_isolated_function_wrapper(): + source = WasmtimeQuickJSCodeRunnerType( + quickjs_wasm_path="/runtime/qjs.wasm" + )._runner_source() + + # User code is executed inside Function("context", ...), not via eval on + # the wrapper's own scope. + assert '_createFunction("context"' in source + assert "globalThis.eval(input.code)" not in source + + # Bridge globals injected by qjs --std are deleted before user code runs. + assert "delete globalThis[_name]" in source + for bridge_global in ("std", "os", "bjson", "print", "console"): + assert f'"{bridge_global}"' in source + + # ECMAScript intrinsics are intentionally NOT shadowed — that pretense was + # bypassable in one line and only created a false sense of security. The + # real boundary is Wasmtime + WASI (see _runner_source docstring). + assert "globalThis.eval = undefined" not in source + assert "globalThis.Function = undefined" not in source + + +@pytest.mark.skipif( + not runtime_variables_are_configured, + reason="Code runner runtime environment variables are not configured.", +) +def test_wasmtime_quickjs_code_runner_executes_real_javascript(): + runner = WasmtimeQuickJSCodeRunnerType( + wasmtime_executable=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE" + ], + quickjs_wasm_path=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH" + ], + ) + + result = runner.run( + {"value": 21}, + """ +function main(context) { + return { + newValue: context.value * 2, + } +} +""", + ) + + assert result == {"newValue": 42} + + +@pytest.mark.skipif( + not runtime_variables_are_configured, + reason="Code runner runtime environment variables are not configured.", +) +def test_wasmtime_quickjs_code_runner_deletes_host_bridge_globals(): + """ + Verify that qjs --std bridge globals are unreachable from user code, even + after a constructor-escape back to the real globalThis. The bridge globals + are the actual security-relevant scrubbing; ECMAScript intrinsics + (Function, eval, globalThis) are intentionally left reachable since they + grant no host access on their own. + """ + + runner = WasmtimeQuickJSCodeRunnerType( + wasmtime_executable=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE" + ], + quickjs_wasm_path=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH" + ], + ) + + result = runner.run( + {}, + """ +function main() { + // Walk back to the real globalThis via constructor escape to confirm the + // bridge globals are not reachable even there. + const realGlobal = (function () {}).constructor("return globalThis")(); + return { + std: typeof realGlobal.std, + os: typeof realGlobal.os, + bjson: typeof realGlobal.bjson, + print: typeof realGlobal.print, + console: typeof realGlobal.console, + scriptArgs: typeof realGlobal.scriptArgs, + execArgv: typeof realGlobal.execArgv, + argv0: typeof realGlobal.argv0, + navigator: typeof realGlobal.navigator, + atob: typeof realGlobal.atob, + btoa: typeof realGlobal.btoa, + // wrapper-local closure refs must not leak into user code's scope + input: typeof input, + write: typeof write, + } +} +""", + ) + + assert result == { + "std": "undefined", + "os": "undefined", + "bjson": "undefined", + "print": "undefined", + "console": "undefined", + "scriptArgs": "undefined", + "execArgv": "undefined", + "argv0": "undefined", + "navigator": "undefined", + "atob": "undefined", + "btoa": "undefined", + "input": "undefined", + "write": "undefined", + } + + +@pytest.mark.skipif( + not runtime_variables_are_configured, + reason="Code runner runtime environment variables are not configured.", +) +def test_wasmtime_quickjs_code_runner_does_not_register_std_as_importable_module(): + """ + The bridge-global deletion only contains user code if std/os/bjson are not + available via dynamic ``import()`` either. This guards against a future + qjs.wasm rebuild that registers them as importable modules — which would + silently make the wrapper's scrubbing incomplete. + """ + + runner = WasmtimeQuickJSCodeRunnerType( + wasmtime_executable=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_WASMTIME_EXECUTABLE" + ], + quickjs_wasm_path=os.environ[ + "BASEROW_ENTERPRISE_CODE_RUNNER_QUICKJS_WASM_PATH" + ], + ) + + result = runner.run( + {}, + """ +function main() { + const outcomes = {}; + for (const spec of ["std", "os", "bjson"]) { + let outcome = "pending"; + try { + import(spec).then( + () => { outcome = "loaded"; }, + () => { outcome = "rejected"; } + ); + } catch (e) { + outcome = "threw"; + } + outcomes[spec] = outcome; + } + return outcomes; +} +""", + ) + + # If a future build registered these as modules, at least one of these + # would be "loaded" once the microtask queue drains. The "pending" status + # is acceptable because dynamic-import resolution is async and the + # surrounding main() returns synchronously — but it must NEVER be + # "loaded". + assert "loaded" not in result.values() diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_type.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_type.py new file mode 100644 index 0000000000..ef9c97e245 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_type.py @@ -0,0 +1,229 @@ +import json +from datetime import datetime + +from django.test import override_settings +from django.urls import reverse + +import pytest +from rest_framework.status import HTTP_200_OK + +from baserow.contrib.builder.workflow_actions.models import EventTypes +from baserow.contrib.builder.workflow_actions.service import ( + BuilderWorkflowActionService, +) +from baserow.contrib.builder.workflow_actions.workflow_action_types import ( + NotificationWorkflowActionType, +) +from baserow.core.code_runner.exceptions import ( + CodeRunnerExecutionError, + CodeRunnerResultError, +) +from baserow.core.services.exceptions import ( + ServiceImproperlyConfiguredDispatchException, + UnexpectedDispatchException, +) +from baserow.test_utils.pytest_conftest import FakeDispatchContext +from baserow_enterprise.apps import register_code_runner_features +from baserow_enterprise.builder.workflow_actions.models import CoreCodeWorkflowAction + +pytestmark = pytest.mark.django_db + + +class FakeCodeRunnerType: + def __init__(self, result=None, exception=None): + self.result = result or {"newValue": 4} + self.exception = exception + self.calls = [] + + def run(self, context_data, code): + self.calls.append((context_data, code)) + if self.exception: + raise self.exception + return self.result + + +def unregister_code_runner_features( + builder_workflow_action_registry, + automation_node_type_registry, + service_type_registry, + code_runner_type_registry, +): + builder_workflow_action_registry.registry.pop("code", None) + automation_node_type_registry.registry.pop("code", None) + service_type_registry.registry.pop("code", None) + code_runner_type_registry.registry.pop("wasmtime_quickjs", None) + + +@pytest.fixture +def code_runner_registered( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, +): + unregister_code_runner_features( + mutable_builder_workflow_action_registry, + mutable_automation_node_type_registry, + mutable_service_type_registry, + mutable_code_runner_type_registry, + ) + + with override_settings(ENTERPRISE_CODE_RUNNER_DEFAULT_TYPE="wasmtime_quickjs"): + register_code_runner_features() + yield + + +def test_core_code_service_type_dispatch_resolves_injections( + enterprise_data_fixture, monkeypatch, code_runner_registered +): + service = enterprise_data_fixture.create_enterprise_core_code_service( + code="function main(context) { return { newValue: 4 } }" + ) + service.injections.create(name="value", formula="get('value')") + + code_runner = FakeCodeRunnerType() + monkeypatch.setattr( + "baserow_enterprise.integrations.core.service_types.get_code_runner", + lambda: code_runner, + ) + + dispatch_result = service.get_type().dispatch( + service, + FakeDispatchContext(context={"value": 2}), + ) + + assert dispatch_result.data == {"newValue": 4} + assert code_runner.calls == [ + ( + {"value": 2}, + "function main(context) { return { newValue: 4 } }", + ) + ] + + +def test_core_code_service_type_dispatch_converts_injections_to_json_values( + enterprise_data_fixture, monkeypatch, code_runner_registered +): + service = enterprise_data_fixture.create_enterprise_core_code_service( + code="function main(context) { return { value: context.value } }" + ) + service.injections.create(name="value", formula="get('value')") + + code_runner = FakeCodeRunnerType() + monkeypatch.setattr( + "baserow_enterprise.integrations.core.service_types.get_code_runner", + lambda: code_runner, + ) + + service.get_type().dispatch( + service, + FakeDispatchContext(context={"value": datetime(2024, 12, 17, 12, 0, 0)}), + ) + + assert code_runner.calls == [ + ( + {"value": "2024-12-17T12:00:00"}, + "function main(context) { return { value: context.value } }", + ) + ] + + +def test_core_code_service_type_dispatch_maps_execution_errors( + enterprise_data_fixture, monkeypatch, code_runner_registered +): + service = enterprise_data_fixture.create_enterprise_core_code_service(code="") + code_runner = FakeCodeRunnerType(exception=CodeRunnerExecutionError("boom")) + monkeypatch.setattr( + "baserow_enterprise.integrations.core.service_types.get_code_runner", + lambda: code_runner, + ) + + with pytest.raises(UnexpectedDispatchException, match="boom"): + service.get_type().dispatch(service, FakeDispatchContext()) + + +def test_core_code_service_type_dispatch_maps_result_errors( + enterprise_data_fixture, monkeypatch, code_runner_registered +): + service = enterprise_data_fixture.create_enterprise_core_code_service(code="") + code_runner = FakeCodeRunnerType(exception=CodeRunnerResultError("object required")) + monkeypatch.setattr( + "baserow_enterprise.integrations.core.service_types.get_code_runner", + lambda: code_runner, + ) + + with pytest.raises( + ServiceImproperlyConfiguredDispatchException, match="object required" + ): + service.get_type().dispatch(service, FakeDispatchContext()) + + +def test_core_code_workflow_action_dispatch_returns_formula_used_public_result( + api_client, enterprise_data_fixture, monkeypatch, code_runner_registered +): + user, token = enterprise_data_fixture.create_user_and_token() + enterprise_data_fixture.enable_enterprise() + monkeypatch.setattr( + "baserow_enterprise.builder.workflow_actions.workflow_action_types." + "LicenseHandler.raise_if_workspace_doesnt_have_feature", + lambda *args, **kwargs: None, + ) + + builder = enterprise_data_fixture.create_builder_application(user=user) + page = enterprise_data_fixture.create_builder_page(user=user, builder=builder) + element = enterprise_data_fixture.create_builder_button_element(page=page) + service = enterprise_data_fixture.create_enterprise_core_code_service( + code="function main() { return { publicValue: 42, hiddenValue: 7 } }" + ) + workflow_action = CoreCodeWorkflowAction.objects.create( + page=page, + element=element, + event=EventTypes.CLICK, + service=service, + order=0, + ) + + BuilderWorkflowActionService().create_workflow_action( + user, + NotificationWorkflowActionType(), + page=page, + element=element, + event=EventTypes.CLICK, + title=f"get('previous_action.{workflow_action.id}.publicValue')", + ) + + code_runner = FakeCodeRunnerType( + result={"publicValue": 42, "hiddenValue": 7}, + ) + monkeypatch.setattr( + "baserow_enterprise.integrations.core.service_types.get_code_runner", + lambda: code_runner, + ) + + url = reverse( + "api:builder:workflow_action:dispatch", + kwargs={"workflow_action_id": workflow_action.id}, + ) + response = api_client.post( + url, + { + "metadata": json.dumps( + { + "previous_action": { + "current_dispatch_id": "test-dispatch-id", + } + } + ) + }, + format="json", + HTTP_AUTHORIZATION=f"JWT {token}", + ) + + assert response.status_code == HTTP_200_OK + assert response.json() == {"publicValue": 42} + assert code_runner.calls == [ + ( + {}, + "function main() { return { publicValue: 42, hiddenValue: 7 } }", + ) + ] diff --git a/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_validation.py b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_validation.py new file mode 100644 index 0000000000..394977fbc8 --- /dev/null +++ b/enterprise/backend/tests/baserow_enterprise_tests/integrations/core/test_core_code_service_validation.py @@ -0,0 +1,64 @@ +from django.core.exceptions import ValidationError as DjangoValidationError + +import pytest +from rest_framework.exceptions import ValidationError as DRFValidationError + +from baserow_enterprise.integrations.core.models import ( + CORE_CODE_SERVICE_CODE_MAX_LENGTH, + CoreCodeService, +) +from baserow_enterprise.integrations.core.service_types import CoreCodeServiceType + + +def test_core_code_service_code_model_field_has_max_length_validator(): + code_field = CoreCodeService._meta.get_field("code") + + assert code_field.max_length == CORE_CODE_SERVICE_CODE_MAX_LENGTH + code_field.run_validators("a" * CORE_CODE_SERVICE_CODE_MAX_LENGTH) + + with pytest.raises(DjangoValidationError): + code_field.run_validators("a" * (CORE_CODE_SERVICE_CODE_MAX_LENGTH + 1)) + + +def test_core_code_service_code_serializer_field_has_max_length_validator(): + code_serializer_field = CoreCodeServiceType().serializer_field_overrides["code"] + + assert code_serializer_field.max_length == CORE_CODE_SERVICE_CODE_MAX_LENGTH + assert ( + code_serializer_field.run_validation("a" * CORE_CODE_SERVICE_CODE_MAX_LENGTH) + == "a" * CORE_CODE_SERVICE_CODE_MAX_LENGTH + ) + + with pytest.raises(DRFValidationError): + code_serializer_field.run_validation( + "a" * (CORE_CODE_SERVICE_CODE_MAX_LENGTH + 1) + ) + + +def test_core_code_service_code_request_serializer_field_has_max_length_validator(): + service_type = CoreCodeServiceType() + code_serializer_field = service_type.request_serializer_field_overrides["code"] + + assert code_serializer_field.max_length == CORE_CODE_SERVICE_CODE_MAX_LENGTH + assert ( + code_serializer_field.run_validation("a" * CORE_CODE_SERVICE_CODE_MAX_LENGTH) + == "a" * CORE_CODE_SERVICE_CODE_MAX_LENGTH + ) + + with pytest.raises(DRFValidationError): + code_serializer_field.run_validation( + "a" * (CORE_CODE_SERVICE_CODE_MAX_LENGTH + 1) + ) + + +def test_core_code_service_request_serializer_validates_code_length(): + serializer_class = CoreCodeServiceType().get_serializer_class( + request_serializer=True + ) + + serializer = serializer_class( + data={"code": "a" * (CORE_CODE_SERVICE_CODE_MAX_LENGTH + 1)} + ) + + assert not serializer.is_valid() + assert "code" in serializer.errors diff --git a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss index 6239ef9ae8..d1ca879848 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss +++ b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/all.scss @@ -21,6 +21,7 @@ @import 'file_input_element'; @import 'file_input_element_form'; @import 'custom_code'; +@import 'core_code_service_form'; @import 'assistant'; @import 'assistant_onboarding'; @import 'date_dependency'; diff --git a/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/core_code_service_form.scss b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/core_code_service_form.scss new file mode 100644 index 0000000000..f967b1b278 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/assets/scss/components/core_code_service_form.scss @@ -0,0 +1,26 @@ +.core-code-service-form__injections-header, +.core-code-service-form__injection { + display: flex; + gap: 6px; +} + +.core-code-service-form__injection { + align-items: flex-start; + flex-wrap: wrap; +} + +.core-code-service-form__injection-name { + flex: 0 0 calc(30% - 3px); +} + +.core-code-service-form__injection-formula { + flex: 1; +} + +.core-code-service-form__injection-delete { + flex: 0; +} + +.core-code-service-form__injection-error { + flex: 0 0 100%; +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/automation/nodeTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/automation/nodeTypes.js new file mode 100644 index 0000000000..abada5ddd0 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/automation/nodeTypes.js @@ -0,0 +1,31 @@ +import { NodeType } from '@baserow/modules/automation/nodeTypes' +import { ActionNodeTypeMixin } from '@baserow/modules/automation/nodeTypeMixins' +import { CoreCodeServiceType } from '@baserow_enterprise/integrations/core/serviceTypes' +import EnterpriseFeaturesObject from '@baserow_enterprise/features' + +export class CoreCodeNodeType extends ActionNodeTypeMixin(NodeType) { + static getType() { + return 'code' + } + + getOrder() { + return 6 + } + + get name() { + return this.app.$i18n.t('nodeType.codeLabel') + } + + get serviceType() { + return this.app.$registry.get('service', CoreCodeServiceType.getType()) + } + + isDeactivatedReason({ workspace }) { + if ( + !this.app.$hasFeature(EnterpriseFeaturesObject.CODE_RUNNER, workspace.id) + ) { + return this.app.$i18n.t('enterprise.deactivated') + } + return super.isDeactivatedReason({ workspace }) + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/builder/workflowActionTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/builder/workflowActionTypes.js new file mode 100644 index 0000000000..754cd8d788 --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/builder/workflowActionTypes.js @@ -0,0 +1,26 @@ +import { WorkflowActionServiceType } from '@baserow/modules/builder/workflowActionTypes' +import { CoreCodeServiceType } from '@baserow_enterprise/integrations/core/serviceTypes' +import EnterpriseFeaturesObject from '@baserow_enterprise/features' + +export class CoreCodeWorkflowActionType extends WorkflowActionServiceType { + static getType() { + return 'code' + } + + get serviceType() { + return this.app.$registry.get('service', CoreCodeServiceType.getType()) + } + + getOrder() { + return 9 + } + + isDeactivatedReason({ workspace }) { + if ( + !this.app.$hasFeature(EnterpriseFeaturesObject.CODE_RUNNER, workspace.id) + ) { + return this.app.$i18n.t('enterprise.deactivated') + } + return super.isDeactivatedReason({ workspace }) + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/components/EnterpriseFeatures.vue b/enterprise/web-frontend/modules/baserow_enterprise/components/EnterpriseFeatures.vue index 8cd7efa1d3..1409d81d1f 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/components/EnterpriseFeatures.vue +++ b/enterprise/web-frontend/modules/baserow_enterprise/components/EnterpriseFeatures.vue @@ -63,6 +63,7 @@ diff --git a/enterprise/web-frontend/modules/baserow_enterprise/integrations/core/serviceTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/integrations/core/serviceTypes.js new file mode 100644 index 0000000000..0b3b34880e --- /dev/null +++ b/enterprise/web-frontend/modules/baserow_enterprise/integrations/core/serviceTypes.js @@ -0,0 +1,63 @@ +import { + ServiceType, + WorkflowActionServiceTypeMixin, +} from '@baserow/modules/core/serviceTypes' +import CoreCodeServiceForm from '@baserow_enterprise/integrations/core/components/services/CoreCodeServiceForm.vue' + +export const CORE_CODE_SERVICE_DEFAULT_CODE = `function main(context) { + return { + message: 'Hello from Baserow', + } +}` + +export class CoreCodeServiceType extends WorkflowActionServiceTypeMixin( + ServiceType +) { + static getType() { + return 'code' + } + + get icon() { + return 'iconoir-code' + } + + get name() { + return this.app.$i18n.t('serviceType.coreCode') + } + + get description() { + return this.app.$i18n.t('serviceType.coreCodeDescription') + } + + getDefaultValues(service, values) { + const defaultValues = super.getDefaultValues(service, values) + if (!defaultValues.code) { + return { + ...defaultValues, + code: CORE_CODE_SERVICE_DEFAULT_CODE, + } + } + + return defaultValues + } + + getErrorMessage({ service }) { + if (service !== undefined && service.code !== undefined && !service.code) { + return this.app.$i18n.t('serviceType.errorCodeMissing') + } + + return super.getErrorMessage({ service }) + } + + getDataSchema(service) { + return service.schema + } + + get formComponent() { + return CoreCodeServiceForm + } + + getOrder() { + return 4 + } +} diff --git a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js index 6c806b221b..bf1d698c65 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/licenseTypes.js @@ -112,6 +112,7 @@ export class EnterpriseWithoutSupportLicenseType extends AdvancedLicenseType { ...commonAdvancedFeatures, EnterpriseFeaturesObject.ENTERPRISE_SETTINGS, EnterpriseFeaturesObject.DATA_SCANNER, + EnterpriseFeaturesObject.CODE_RUNNER, ] } diff --git a/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json b/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json index d1d710083e..4f7537ee03 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json +++ b/enterprise/web-frontend/modules/baserow_enterprise/locales/en.json @@ -19,6 +19,29 @@ "trashType": { "team": "team" }, + "serviceType": { + "coreCode": "Execute code", + "coreCodeDescription": "Runs arbitrary JavaScript code.", + "errorCodeMissing": "Missing code property" + }, + "coreCodeServiceForm": { + "codeLabel": "Code", + "codeHelperText": "Baserow calls the main(context) function when the action runs. Use context to read injected values and return an object whose properties can be used by later actions.", + "codePlaceholder": "Enter the code to execute", + "editCode": "Edit code", + "injectionsLabel": "Data injections", + "injectionsHelperText": "Injections are formulas evaluated before the code runs. Each injection name becomes a property of context. For example an injection named customerName is available as context.customerName.", + "nameLabel": "Name", + "namePlaceholder": "Variable name", + "valueLabel": "Value", + "valuePlaceholder": "Value formula", + "addInjection": "Add injection", + "nameFieldRequired": "The name field is required.", + "nameFieldInvalid": "The name must be a valid variable name." + }, + "nodeType": { + "codeLabel": "Execute code" + }, "createTeamModal": { "title": "Create team", "invalidNameTitle": "Please use a different name", diff --git a/enterprise/web-frontend/modules/baserow_enterprise/module.js b/enterprise/web-frontend/modules/baserow_enterprise/module.js index 7c806838b7..c3c6f27d4c 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/module.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/module.js @@ -89,6 +89,7 @@ export default defineNuxtModule({ nuxt.options.runtimeConfig.public, { baserowEnterpriseAssistantLlmModel: '', + baserowEnterpriseCodeRunnerDefaultType: 'wasmtime_quickjs', baserowExtraClientScriptUrls: '', } ) diff --git a/enterprise/web-frontend/modules/baserow_enterprise/plugin.js b/enterprise/web-frontend/modules/baserow_enterprise/plugin.js index c5e16620fd..57b86b4ea9 100644 --- a/enterprise/web-frontend/modules/baserow_enterprise/plugin.js +++ b/enterprise/web-frontend/modules/baserow_enterprise/plugin.js @@ -93,12 +93,15 @@ import { CustomCodeBuilderSettingType } from '@baserow_enterprise/builderSetting import { RealtimePushTwoWaySyncStrategyType } from '@baserow_enterprise/twoWaySyncStrategyTypes' import { RestrictedViewOwnershipType } from '@baserow_enterprise/viewOwnershipTypes' import { AIDatabaseOnboardingStepType } from '@baserow_enterprise/databaseOnboardingStepTypes' +import { CoreCodeServiceType } from '@baserow_enterprise/integrations/core/serviceTypes' +import { CoreCodeWorkflowActionType } from '@baserow_enterprise/builder/workflowActionTypes' +import { CoreCodeNodeType } from '@baserow_enterprise/automation/nodeTypes' export default defineNuxtPlugin({ name: 'enterprise', dependsOn: ['premium', 'registry'], setup(nuxtApp) { - const { $registry, $store } = nuxtApp + const { $config, $registry, $store } = nuxtApp const context = { app: nuxtApp } @@ -157,6 +160,14 @@ export default defineNuxtPlugin({ $registry.register('license', new EnterpriseLicenseType(context)) $registry.register('userSource', new LocalBaserowUserSourceType(context)) + if ($config.public.baserowEnterpriseCodeRunnerDefaultType) { + $registry.register('service', new CoreCodeServiceType(context)) + $registry.register( + 'workflowAction', + new CoreCodeWorkflowActionType(context) + ) + $registry.register('node', new CoreCodeNodeType(context)) + } $registry.register( 'appAuthProvider', diff --git a/enterprise/web-frontend/test/unit/enterprise/integrations/core/coreCodeServiceForm.spec.js b/enterprise/web-frontend/test/unit/enterprise/integrations/core/coreCodeServiceForm.spec.js new file mode 100644 index 0000000000..c9f8db3336 --- /dev/null +++ b/enterprise/web-frontend/test/unit/enterprise/integrations/core/coreCodeServiceForm.spec.js @@ -0,0 +1,84 @@ +import { defineComponent, nextTick } from 'vue' +import { mountSuspended } from '@nuxt/test-utils/runtime' + +import CoreCodeServiceForm from '@baserow_enterprise/integrations/core/components/services/CoreCodeServiceForm' + +const CORE_CODE_SERVICE_CODE_MAX_LENGTH = 4096 + +const FormGroupStub = defineComponent({ + name: 'FormGroup', + props: { + error: { + type: Boolean, + default: false, + }, + errorMessage: { + type: String, + default: '', + }, + }, + template: + '
', +}) + +const CodeEditorStub = defineComponent({ + name: 'CodeEditor', + props: { + modelValue: { + type: String, + default: '', + }, + }, + emits: ['update:modelValue'], + template: + '