diff --git a/.github/workflows/knowledge-fs-ci.yml b/.github/workflows/knowledge-fs-ci.yml index 0b2757f47a8ea7..b4fc2ce288fe34 100644 --- a/.github/workflows/knowledge-fs-ci.yml +++ b/.github/workflows/knowledge-fs-ci.yml @@ -95,7 +95,7 @@ jobs: - 'api/tests/unit_tests/core/workflow/test_node_runtime.py' - 'api/tests/unit_tests/services/enterprise/test_rbac_service.py' - 'api/tests/unit_tests/repositories/workspace/test_workspace_repository.py' - - 'api/tests/unit_tests/services/test_agent_tool_inner_service.py' + - 'api/tests/unit_tests/services/agent/test_tool_invocation_service.py' - 'api/tests/unit_tests/services/test_vector_space_admission_service.py' - 'api/tests/unit_tests/services/test_knowledge_fs_capability.py' - 'api/tests/unit_tests/services/knowledge_fs/test_product_operations.py' @@ -275,7 +275,7 @@ jobs: api/tests/unit_tests/core/workflow/test_structured_output_compatibility.py api/tests/unit_tests/services/enterprise/test_rbac_service.py api/tests/unit_tests/repositories/workspace/test_workspace_repository.py - api/tests/unit_tests/services/test_agent_tool_inner_service.py + api/tests/unit_tests/services/agent/test_tool_invocation_service.py api/tests/unit_tests/services/test_vector_space_admission_service.py api/tests/unit_tests/services/test_tidb_binding_service.py ) diff --git a/api/.importlinter b/api/.importlinter index c55841d85722e2..8b933a6c06f5ab 100644 --- a/api/.importlinter +++ b/api/.importlinter @@ -32,6 +32,8 @@ layers = # error makes this list shrink whenever a legacy dependency is removed. unmatched_ignore_imports_alerting = error ignore_imports = + core.app.apps.advanced_chat.app_runner -> extensions.workflow_warm_shutdown + core.app.apps.workflow.app_runner -> extensions.workflow_warm_shutdown # The Dify node factory composes the Human Input runtime for workflow execution. core.workflow.node_factory -> services.human_input_v2.composition # Legacy generators delegate configured file defaults to the application owner during core migration. @@ -76,7 +78,6 @@ ignore_imports = core.workflow.workflow_entry -> services.workflow_node_variables core.app.apps.workflow_app_runner -> tasks.mail_human_input_delivery_task core.app.features.annotation_reply.annotation_reply -> services.annotation_service - core.app.features.annotation_reply.annotation_reply -> services.knowledge.dataset_service core.app.llm.quota -> services.credit_pool_service core.app.task_pipeline.message_cycle_manager -> services.annotation_service core.helper.credential_utils -> services.enterprise.plugin_manager_service @@ -105,7 +106,6 @@ ignore_imports = # The KnowledgeFS retrieval node is owned by services.knowledge_fs and only wired in here. core.workflow.node_factory -> services.knowledge_fs.knowledge_retrieval_v2.automatic_metadata_filter core.workflow.node_factory -> services.knowledge_fs.knowledge_retrieval_v2.entities - core.workflow.nodes.agent.message_transformer -> services.tools.legacy_builtin_tools_manage_service core.workflow.nodes.agent_v2.agent_node -> services.agent.knowledge_citations core.workflow.nodes.agent_v2.agent_node -> services.agent.prompt_mentions core.workflow.nodes.agent_v2.agent_node -> services.agent.workspace_service @@ -134,7 +134,6 @@ ignore_imports = libs.oauth_bearer -> models libs.rsa -> extensions.ext_storage libs.workspace_permission -> services.enterprise.enterprise_service - services.app_generate_service -> controllers.console.app.workflow [importlinter:contract:no-direct-rsa-imports] # Note: `libs` itself is deliberately excluded from source_modules -- import-linter's diff --git a/api/celery_healthcheck.py b/api/celery_healthcheck.py index 23d856d7d04dc0..b4c3d3ecd36331 100644 --- a/api/celery_healthcheck.py +++ b/api/celery_healthcheck.py @@ -1,18 +1,18 @@ -# This module provides a lightweight Celery instance for use in Docker health checks. -# Unlike celery_entrypoint.py, this does NOT import app.py and therefore avoids -# initializing all Flask extensions (DB, Redis, storage, blueprints, etc.). -# Using this module keeps the health check fast and low-cost. -from celery import Celery - -from configs import dify_config -from extensions.ext_celery import get_celery_broker_transport_options, get_celery_ssl_options - -celery = Celery(broker=dify_config.CELERY_BROKER_URL) - -broker_transport_options = get_celery_broker_transport_options() -if broker_transport_options: - celery.conf.update(broker_transport_options=broker_transport_options) - -ssl_options = get_celery_ssl_options() -if ssl_options: - celery.conf.update(broker_use_ssl=ssl_options) +# This module provides a lightweight Celery instance for use in Docker health checks. +# Unlike celery_entrypoint.py, this does NOT import app.py and therefore avoids +# initializing all Flask extensions (DB, Redis, storage, blueprints, etc.). +# Using this module keeps the health check fast and low-cost. +from celery import Celery + +from configs import dify_config +from extensions.ext_celery import get_celery_broker_transport_options, get_celery_ssl_options + +celery = Celery(broker=dify_config.CELERY_BROKER_URL) + +broker_transport_options = get_celery_broker_transport_options() +if broker_transport_options: + celery.conf.update(broker_transport_options=broker_transport_options) + +ssl_options = get_celery_ssl_options() +if ssl_options: + celery.conf.update(broker_use_ssl=ssl_options) diff --git a/api/commands/data_migration.py b/api/commands/data_migration.py index 6bd36ee2200473..5ee67eabc94e51 100644 --- a/api/commands/data_migration.py +++ b/api/commands/data_migration.py @@ -23,6 +23,7 @@ migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401 ) from core.db.session_factory import session_factory +from extensions.ext_application_services import application_services from extensions.ext_database import db from graphon.model_runtime.entities.model_entities import ModelType from models import Tenant @@ -290,7 +291,10 @@ def export_migration_data(input_file: str | None, output_file: str | None, overw raw_config = _load_json_object(input_file, "Export config") selection = ExportConfigParser().parse(raw_config) with session_factory.create_session() as session: - result = MigrationExportService().export(selection, session=session) + result = MigrationExportService( + workflow_tools=application_services().tools.workflows, + tool_providers=application_services().tools.tool_providers, + ).export(selection, session=session) MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite) click.echo(click.style(f"Output written to {output_file}", fg="green")) _render_report(result.report_items, context=_with_output_path(result.report_context, output_file)) @@ -338,7 +342,13 @@ def import_migration_data( assert input_file is not None package = MigrationPackageService().load_package(input_file) with session_factory.create_session() as session: - result = MigrationImportService().import_package( + from extensions.application_services.workflow import build_app_dsl_service + + result = MigrationImportService( + workflow_tools=application_services().tools.workflows, + workflows=application_services().console_workflows, + app_dsl=build_app_dsl_service, + ).import_package( ImportRequest( package=package, cli_target_tenant=target_tenant, @@ -437,7 +447,10 @@ def migration_data_wizard() -> None: output_file=output_file, ) with session_factory.create_session() as session: - result = MigrationExportService().export(selection, session=session) + result = MigrationExportService( + workflow_tools=application_services().tools.workflows, + tool_providers=application_services().tools.tool_providers, + ).export(selection, session=session) MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite) click.echo(click.style(f"Output written to {output_file}", fg="green")) _print_wizard_step("Report") diff --git a/api/controllers/console/agent/composer.py b/api/controllers/console/agent/composer.py index fb2446f5fe845c..a6ebe298f4fb8c 100644 --- a/api/controllers/console/agent/composer.py +++ b/api/controllers/console/agent/composer.py @@ -2,8 +2,8 @@ from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import NotFound +from controllers.common.errors import NotFoundError from controllers.common.rbac import AgentId, PlainApp, RBACCheck, Workspace from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.common.session import with_session @@ -19,6 +19,8 @@ with_current_tenant_id, with_current_user_id, ) +from extensions.application_services.snippets import build_snippet_service +from extensions.ext_application_services import application_services from fields.agent_fields import ( AgentAppComposerResponse, AgentComposerCandidatesResponse, @@ -46,7 +48,6 @@ WorkflowAgentComposerQuery, WorkflowComposerCopyFromRosterPayload, ) -from services.snippet_service import SnippetService register_schema_models( console_ns, ComposerSavePayload, WorkflowAgentComposerQuery, WorkflowComposerCopyFromRosterPayload @@ -225,11 +226,13 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, app_model: return dump_response( AgentComposerCandidatesResponse, AgentComposerService.get_workflow_candidates( + variables=application_services().workflow_variables, session=session, tenant_id=tenant_id, app_id=app_model.id, node_id=node_id, user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) @@ -307,12 +310,12 @@ def post( def _require_snippet_app_id(*, session: Session, tenant_id: str, snippet_id: UUID) -> str: - snippet = SnippetService(session=session).get_snippet_by_id( + snippet = build_snippet_service(session=session).get_snippet_by_id( snippet_id=str(snippet_id), # pyrefly: ignore[unnecessary-type-conversion] tenant_id=tenant_id, ) if snippet is None: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") return snippet.id @@ -470,11 +473,13 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, snippet_id return dump_response( AgentComposerCandidatesResponse, AgentComposerService.get_workflow_candidates( + variables=application_services().workflow_variables, session=session, tenant_id=tenant_id, app_id=_require_snippet_app_id(session=session, tenant_id=tenant_id, snippet_id=snippet_id), node_id=node_id, user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) @@ -650,5 +655,6 @@ def get(self, session: Session, tenant_id: str, current_user_id: str, agent_id: tenant_id=tenant_id, agent_id=str(agent_id), user_id=current_user_id, + tool_providers=application_services().tools.tool_providers, ), ) diff --git a/api/controllers/console/agent/roster.py b/api/controllers/console/agent/roster.py index 8a6a0d0331b3d7..b7e5f0ab500408 100644 --- a/api/controllers/console/agent/roster.py +++ b/api/controllers/console/agent/roster.py @@ -1,7 +1,7 @@ from typing import Literal from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource from pydantic import AliasChoices, BaseModel, Field, field_validator from sqlalchemy import func, or_, select @@ -9,6 +9,7 @@ from configs import dify_config from controllers.common.agent_access import resolve_agent_access_filter +from controllers.common.errors import AccessDeniedError, InvalidRequestError from controllers.common.rbac import AgentId, RBACCheck, Workspace from controllers.common.schema import ( query_params_from_model, @@ -431,7 +432,14 @@ def _serialize_agent_app_detail( roster_service = _agent_roster_service(session) payload = GenericAppDetailWithSite.model_validate( - AppResponseView(app_model, session=session, account=current_user, access_mode=access_mode), + AppResponseView( + app_model, + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, + session=session, + account=current_user, + access_mode=access_mode, + ), from_attributes=True, ).model_dump(mode="json") agent = ( @@ -524,7 +532,15 @@ def _serialize_agent_app_pagination( "limit": app_pagination.per_page, "total": app_pagination.total, "has_more": app_pagination.has_next, - "data": [AppResponseView(app, session=session) for app in app_pagination.items], + "data": [ + AppResponseView( + app, + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, + session=session, + ) + for app in app_pagination.items + ], }, ).model_dump(mode="json") payload["publication_counts"] = { @@ -623,7 +639,7 @@ def _parse_observability_time_range(start: str | None, end: str | None, account: try: return parse_time_range(start, end, timezone) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) from exc def _query_values(name: str, alias_name: str | None = None) -> list[str]: @@ -1210,7 +1226,7 @@ def get(self, session: Session, tenant_id: str, current_user: Account, agent_id: ), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) from exc return dump_response(AgentLogListResponse, payload) @@ -1251,7 +1267,7 @@ def get(self, session: Session, tenant_id: str, current_user: Account, agent_id: ), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) from exc return dump_response(AgentLogMessageListResponse, payload) @@ -1307,7 +1323,7 @@ def get( params=AgentStatisticsQueryParams(source=req_data.source, start=start, end=end, timezone=timezone), ) except ValueError as exc: - abort(400, description=str(exc)) + raise InvalidRequestError(str(exc)) from exc return dump_response(AgentStatisticSummaryEnvelopeResponse, payload) @@ -1366,7 +1382,7 @@ def post(self, session: Session, tenant_id: str, current_user: Account, agent_id dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD and not FeatureService.get_workspace_plan(tenant_id).is_paid ): - abort(403, description="This feature requires a paid plan.") + raise AccessDeniedError("This feature requires a paid plan.") return dump_response( AgentConfigSnapshotRestoreResponse, diff --git a/api/controllers/console/app/completion.py b/api/controllers/console/app/completion.py index b097914b5b8f8e..62329123a452a6 100644 --- a/api/controllers/console/app/completion.py +++ b/api/controllers/console/app/completion.py @@ -8,9 +8,9 @@ from flask_restx import Resource from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import SimpleResultResponse from controllers.common.rbac import AgentId, PlainApp, RBACCheck from controllers.common.schema import register_response_schema_models, register_schema_models @@ -45,6 +45,7 @@ QuotaExceededError, ) from core.helper.trace_id_helper import get_external_trace_id +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.external_api import error_responses @@ -74,7 +75,7 @@ def _resolve_debugger_chat_streaming( if app_mode != AppMode.AGENT: return response_mode != "blocking" if response_mode_provided and response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -178,6 +179,8 @@ def post(self, req_data: CompletionMessagePayload, session: Session, current_use try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=current_user, @@ -189,7 +192,7 @@ def post(self, req_data: CompletionMessagePayload, session: Session, current_use # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: @@ -420,7 +423,7 @@ def _resolve_current_user_agent_debug_conversation_id( account_id=current_user.id, ) if conversation_id is None: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") return conversation_id @@ -450,7 +453,7 @@ def _create_chat_message( start_new=draft_type == AgentConfigDraftType.DRAFT and not args_model.conversation_id, ) if args_model.conversation_id and args_model.conversation_id != debug_conversation_id: - raise NotFound("Conversation Not Exists.") + raise NotFoundError("Conversation Not Exists.") args["conversation_id"] = debug_conversation_id streaming = _resolve_debugger_chat_streaming( @@ -562,6 +565,8 @@ def _generate_chat_message( ): try: return AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=current_user, @@ -570,7 +575,7 @@ def _generate_chat_message( streaming=streaming, ) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: diff --git a/api/controllers/console/app/model_config.py b/api/controllers/console/app/model_config.py index 10673553d19769..01a34d1275fe1b 100644 --- a/api/controllers/console/app/model_config.py +++ b/api/controllers/console/app/model_config.py @@ -20,13 +20,14 @@ with_current_user_id, ) from core.agent.entities import AgentToolEntity -from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager from events.app_event import app_model_config_was_updated +from extensions.ext_application_services import application_services from libs.datetime_utils import naive_utc_now from libs.login import login_required from models.model import App, AppMode, AppModelConfig from services.app_model_config_service import AppModelConfigService +from services.tools.tool_manager import ToolManager register_schema_models(console_ns, AppModelConfigPayload) register_response_schema_models(console_ns, SimpleResultResponse) @@ -89,6 +90,8 @@ def post(self, session: Session, current_tenant_id: str, current_user_id: str, a # get tool try: tool_runtime = ToolManager.get_agent_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, tenant_id=current_tenant_id, app_id=app_model.id, agent_tool=agent_tool_entity, @@ -129,6 +132,8 @@ def post(self, session: Session, current_tenant_id: str, current_user_id: str, a else: try: tool_runtime = ToolManager.get_agent_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, tenant_id=current_tenant_id, app_id=app_model.id, agent_tool=agent_tool_entity, diff --git a/api/controllers/console/app/workflow.py b/api/controllers/console/app/workflow.py index b436f81cd093cf..b400ab98a5a3b6 100644 --- a/api/controllers/console/app/workflow.py +++ b/api/controllers/console/app/workflow.py @@ -1,30 +1,23 @@ import json import logging -from collections.abc import Sequence -from datetime import datetime -from typing import Any, Literal, NotRequired, Self, TypedDict +from typing import Any, Literal, Self +from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource -from pydantic import ( - AliasChoices, - BaseModel, - ConfigDict, - Field, - RootModel, - field_validator, - model_validator, -) +from pydantic import BaseModel, ConfigDict, Field, RootModel, field_validator, model_validator from pydantic.json_schema import SkipJsonSchema -from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound -import services -from configs import dify_config +import services.errors.conversation from controllers.common.controller_schemas import DefaultBlockConfigQuery, WorkflowListQuery, WorkflowUpdatePayload -from controllers.common.errors import InvalidArgumentError +from controllers.common.errors import ( + InternalServerError, + InvalidArgumentError, + InvalidRequestError, + NotFoundError, + UnsupportedMediaTypeError, +) from controllers.common.fields import GeneratedAppResponse, NewAppResponse, SimpleResultResponse -from controllers.common.rbac import PlainApp, RBACCheck from controllers.common.schema import ( query_params_from_model, register_response_schema_model, @@ -32,96 +25,47 @@ register_schema_models, ) from controllers.console import console_ns -from controllers.console.app.error import ( - ConversationCompletedError, - DraftWorkflowNotExist, - DraftWorkflowNotSync, -) -from controllers.console.app.permission_keys import get_app_permission_keys -from controllers.console.app.wraps import get_app_model, with_session -from controllers.console.wraps import ( - RBACPermission, - account_initialization_required, - edit_permission_required, - model_validate, - rbac_permission_required, - setup_required, - with_current_tenant_id, - with_current_user, -) +from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync +from controllers.console.app.workflow_admission import console_workflow_admission +from controllers.console.flask_admission import console_account_admission +from controllers.console.wraps import RBACPermission, model_validate from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.app.entities.app_invoke_entities import InvokeFrom -from core.app.file_access import DatabaseFileAccessController -from core.db.session_factory import session_factory -from core.helper import encrypter from core.helper.trace_id_helper import get_external_trace_id -from core.plugin.impl.exc import PluginInvokeError -from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE -from core.trigger.debug.event_selectors import ( - TriggerDebugEvent, - TriggerDebugEventPoller, - create_event_poller, - select_trigger_debug_events, -) -from core.workflow.llm_environment_variable import ( - LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE, - LLMEnvironmentVariable, -) -from extensions.ext_database import db -from extensions.ext_redis import redis_client -from factories import file_factory, variable_factory +from extensions.ext_application_services import application_services from fields.base import ResponseModel from fields.conversation_variable_fields import WorkflowConversationVariableResponse -from fields.member_fields import SimpleAccount -from fields.workflow_run_fields import ( - WorkflowRunNodeExecutionResponse, - node_execution_response_source, +from fields.workflow_fields import ( + DefaultBlockConfigResponse, + DefaultBlockConfigsResponse, + PipelineVariableResponse, + WorkflowEnvironmentVariableResponse, + WorkflowPaginationResponse, + WorkflowPublishResponse, + WorkflowResponse, + WorkflowRestoreResponse, ) -from graphon.enums import NodeType -from graphon.file import File -from graphon.file import helpers as file_helpers +from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse from graphon.model_runtime.utils.encoders import jsonable_encoder -from graphon.variables import SecretVariable, SegmentType, VariableBase from graphon.variables.exc import VariableError from libs import helper -from libs.datetime_utils import naive_utc_now from libs.external_api import error_responses -from libs.helper import TimestampField, dump_response, to_timestamp, uuid_value -from libs.login import login_required -from models import Account, App +from libs.helper import TimestampField, dump_response, uuid_value +from machinery.context import RequestContext from models.human_input_entities import FormInputConfig, UserActionConfig from models.model import AppMode -from models.workflow import Workflow -from repositories.workflow_collaboration_repository import WORKFLOW_ONLINE_USERS_PREFIX -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.app.access import resolve_app_access_filter -from services.app_generate_service import AppGenerateService -from services.app_task_service import AppTaskService from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.llm import InvokeRateLimitError -from services.workflow_ref_service import WorkflowRefService -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService -from services.workflow_variable_reference_validator import advisory_variable_reference_warning +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, + WorkflowTriggerError, +) logger = logging.getLogger(__name__) - -_file_access_controller = DatabaseFileAccessController() LISTENING_RETRY_IN = 2000 -RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE = "source workflow must be published" -MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS = 1000 -WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE = 50 -ENVIRONMENT_VARIABLE_SUPPORTED_TYPES = (SegmentType.STRING, SegmentType.NUMBER, SegmentType.SECRET) - - -class EnvironmentVariableResponseDict(TypedDict): - value_type: str - id: NotRequired[str] - name: NotRequired[str] - value: NotRequired[Any] - description: NotRequired[str | None] - class SyncEnvironmentVariablePatchPayload(BaseModel): environment_variables: list[dict[str, Any]] = Field(default_factory=list) @@ -291,108 +235,6 @@ class WorkflowTriggerRunErrorResponse(ResponseModel): error: str | SkipJsonSchema[None] = None -class PipelineVariableResponse(ResponseModel): - label: str - variable: str - type: str - belong_to_node_id: str - max_length: int | None = None - required: bool - unit: str | None = None - default_value: Any = Field(default=None) - options: list[str] | None = None - placeholder: str | None = None - tooltips: str | None = None - allowed_file_types: list[str] | None = None - allowed_file_extensions: list[str] | None = Field( - default=None, validation_alias=AliasChoices("allowed_file_extensions", "allow_file_extension") - ) - allowed_file_upload_methods: list[str] | None = Field( - default=None, validation_alias=AliasChoices("allowed_file_upload_methods", "allow_file_upload_methods") - ) - - -class WorkflowEnvironmentVariableResponse(ResponseModel): - value_type: str - id: str - name: str - value: Any - description: str - - -class WorkflowResponse(ResponseModel): - id: str - graph: dict[str, Any] = Field( - validation_alias=AliasChoices("graph_dict", "graph"), - ) - features: dict[str, Any] = Field( - validation_alias=AliasChoices("features_dict", "features"), - ) - hash: str = Field(validation_alias=AliasChoices("unique_hash", "hash")) - version: str - # NULL for drafts and for versions published before numbering was introduced; those - # render as "Untitled Version" instead of `#N`. Never 0, so clients must test for null. - version_number: int | None = None - marked_name: str - marked_comment: str - created_by: SimpleAccount | None = Field( - default=None, validation_alias=AliasChoices("created_by_account", "created_by") - ) - created_at: int - updated_by: SimpleAccount | None = Field( - default=None, validation_alias=AliasChoices("updated_by_account", "updated_by") - ) - updated_at: int - tool_published: bool - environment_variables: list[WorkflowEnvironmentVariableResponse] - conversation_variables: list[WorkflowConversationVariableResponse] - rag_pipeline_variables: list[PipelineVariableResponse] - - @field_validator("created_at", "updated_at", mode="before") - @classmethod - def _normalize_timestamp(cls, value: datetime | int | None) -> int: - timestamp = to_timestamp(value) - if timestamp is None: - raise ValueError("timestamp is required") - return timestamp - - @field_validator("environment_variables", mode="before") - @classmethod - def _serialize_environment_variables(cls, value: Any) -> list[Any]: - if value is None: - return [] - - return [_serialize_environment_variable(item) for item in value] - - -class WorkflowResponseSource: - def __init__(self, workflow: Workflow, *, session: Session) -> None: - self._workflow = workflow - self._session = session - - def __getattr__(self, name: str) -> object: - return getattr(self._workflow, name) # guard-ignore: no-new-getattr -- delegates model fields - - @property - def created_by_account(self) -> Account | None: - return self._workflow.get_created_by_account(session=self._session) - - @property - def updated_by_account(self) -> Account | None: - return self._workflow.get_updated_by_account(session=self._session) - - @property - def tool_published(self) -> bool: - return self._workflow.get_tool_published(session=self._session) - - -class WorkflowPaginationResponse(ResponseModel): - items: list[WorkflowResponse] - page: int - limit: int - has_more: bool - - class WorkflowOnlineUser(ResponseModel): user_id: str username: str @@ -408,40 +250,12 @@ class WorkflowOnlineUsersResponse(ResponseModel): data: list[WorkflowOnlineUsersByApp] -class WorkflowPublishResponse(ResponseModel): - result: str - created_at: int - warning: str | None = Field( - default=None, - description="Advisory warning for variable references that can read a skipped branch. Publish still succeeds.", - ) - - class SyncDraftWorkflowResponse(ResponseModel): result: str hash: str updated_at: int -class WorkflowRestoreResponse(ResponseModel): - result: str - hash: str - updated_at: int - - -class DefaultBlockConfigItemResponse(ResponseModel): - type: str - config: dict[str, Any] - - -class DefaultBlockConfigsResponse(RootModel[list[DefaultBlockConfigItemResponse]]): - pass - - -class DefaultBlockConfigResponse(RootModel[dict[str, Any]]): - root: dict[str, Any] - - class HumanInputFormPreviewResponse(ResponseModel): form_id: str node_id: str @@ -498,7 +312,11 @@ class DraftWorkflowTriggerRunAllPayload(BaseModel): DraftWorkflowTriggerRunPayload, DraftWorkflowTriggerRunAllPayload, ) + + register_response_schema_model(console_ns, WorkflowRunNodeExecutionResponse) + + register_response_schema_models( console_ns, WorkflowTriggerRunErrorResponse, @@ -524,71 +342,6 @@ class DraftWorkflowTriggerRunAllPayload(BaseModel): ) -# TODO(QuantumGhost): Refactor existing node run API to handle file parameter parsing -# at the controller level rather than in the workflow logic. This would improve separation -# of concerns and make the code more maintainable. -def _parse_file(workflow: Workflow, files: list[dict] | None = None) -> Sequence[File]: - files = files or [] - - file_extra_config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) - file_objs: Sequence[File] = [] - if file_extra_config is None: - return file_objs - file_objs = file_factory.build_from_mappings( - mappings=files, - tenant_id=workflow.tenant_id, - config=file_extra_config, - access_controller=_file_access_controller, - ) - return file_objs - - -def _serialize_environment_variable(value: Any) -> EnvironmentVariableResponseDict | Any: - match value: - case LLMEnvironmentVariable(): - return { - "id": value.id, - "name": value.name, - "value": value.value, - "value_type": LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE, - "description": value.description, - } - - case SecretVariable(): - return { - "id": value.id, - "name": value.name, - "value": encrypter.full_mask_token(), - "value_type": value.value_type.value, - "description": value.description, - } - - case VariableBase(): - return { - "id": value.id, - "name": value.name, - "value": value.value, - "value_type": str(value.value_type.exposed_type()), - "description": value.description, - } - - case dict(): - value_type_str = value.get("value_type") - if not isinstance(value_type_str, str): - raise TypeError( - f"unexpected type for value_type field, value={value_type_str}, type={type(value_type_str)}" - ) - if value_type_str == LLM_ENVIRONMENT_VARIABLE_VALUE_TYPE: - return value - value_type = SegmentType(value_type_str).exposed_type() - if value_type not in ENVIRONMENT_VARIABLE_SUPPORTED_TYPES: - raise ValueError(f"Unsupported environment variable value type: {value_type}") - return value - - case _: - return value - - @console_ns.route("/apps//workflows/draft") class DraftWorkflowApi(Resource): @error_responses(DraftWorkflowNotExist) @@ -601,42 +354,13 @@ class DraftWorkflowApi(Resource): console_ns.models[WorkflowResponse.__name__], ) @console_ns.response(404, "Draft workflow not found") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @with_session(write=False) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, session: Session, app_model: App): - """ - Get draft workflow - """ - # fetch draft workflow by app_model - workflow_service = WorkflowService() - workflow = workflow_service.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise DraftWorkflowNotExist() - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - # Return workflow with response-only Agent node job projection so the - # front-end can treat draft graph node data as the editing source. - response = WorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") - response["graph"] = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + return dump_response( + WorkflowResponse, application_services().console_workflows.draft(request_context, str(app_id)) ) - return response @error_responses(DraftWorkflowNotSync) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) @console_ns.doc("sync_draft_workflow") @console_ns.doc(description="Sync draft workflow configuration") @console_ns.expect(console_ns.models[SyncDraftWorkflowPayload.__name__]) @@ -647,15 +371,9 @@ def get(self, session: Session, app_model: App): ) @console_ns.response(400, "Invalid workflow configuration") @console_ns.response(403, "Permission denied") - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - def post(self, current_user: Account, app_model: App): - """ - Sync draft workflow - """ + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def post(self, request_context: RequestContext, app_id: UUID): content_type = request.headers.get("Content-Type", "") - if "application/json" in content_type: payload_data = request.get_json(silent=True) if not isinstance(payload_data, dict): @@ -667,57 +385,38 @@ def post(self, current_user: Account, app_model: App): except ValueError as caught_error: raise InvalidArgumentError("Invalid JSON data") from caught_error else: - abort(415) + raise UnsupportedMediaTypeError() # Empty autosaves can otherwise replace a populated draft, even with a matching hash. # Require explicit confirmation for both regular saves and text/plain unload beacons. if args_model.graph.get("nodes") == [] and args_model.graph.get("edges") == [] and not args_model.force: raise InvalidArgumentError(description="Saving an empty workflow requires force=true.") - workflow_service = WorkflowService() - + patch = args_model.environment_variable_patch try: - environment_variable_patch = args_model.environment_variable_patch - environment_variable_upserts: list[VariableBase] | None = None - deleted_environment_variable_ids: list[str] = [] - if environment_variable_patch is not None: - environment_variable_upsert_mappings = Workflow.normalize_environment_variable_mappings( - environment_variable_patch.environment_variables, - ) - environment_variable_upserts = [ - variable_factory.build_environment_variable_from_mapping(obj) - for obj in environment_variable_upsert_mappings - ] - deleted_environment_variable_ids = environment_variable_patch.deleted_environment_variable_ids - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) - for obj in args_model.conversation_variables - ] - workflow = workflow_service.sync_draft_workflow( - app_model=app_model, - graph=args_model.graph, - features=args_model.features, - unique_hash=args_model.hash, - account=current_user, - environment_variables=[], - conversation_variables=conversation_variables, - session=db.session(), - environment_variable_upserts=environment_variable_upserts, - deleted_environment_variable_ids=deleted_environment_variable_ids, - preserve_environment_variables=True, - graph_only=args_model.is_collaborative, + result = application_services().console_workflows.sync( + request_context, + str(app_id), + DraftSyncCommand( + graph=args_model.graph, + features=args_model.features, + unique_hash=args_model.hash, + is_collaborative=args_model.is_collaborative, + environment_upserts=patch.environment_variables if patch is not None else None, + environment_deletions=patch.deleted_environment_variable_ids if patch is not None else [], + conversation_variables=args_model.conversation_variables, + ), ) - except WorkflowHashNotEqualError as caught_error: - raise DraftWorkflowNotSync() from caught_error - except VariableError as e: - raise InvalidArgumentError(description=str(e)) from e - + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except VariableError as error: + raise InvalidArgumentError(description=str(error)) from error return dump_response( SyncDraftWorkflowResponse, { "result": "success", - "hash": workflow.unique_hash, - "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), + "hash": result.hash, + "updated_at": TimestampField().format(result.updated_at), }, ) @@ -732,47 +431,27 @@ class AdvancedChatDraftWorkflowRunApi(Resource): @console_ns.response(200, "Workflow run started successfully", console_ns.models[GeneratedAppResponse.__name__]) @console_ns.response(400, "Invalid request parameters") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(AdvancedChatWorkflowRunPayload) - def post(self, payload: AdvancedChatWorkflowRunPayload, session: Session, current_user: Account, app_model: App): - """ - Run draft workflow - """ + def post(self, payload: AdvancedChatWorkflowRunPayload, request_context: RequestContext, app_id: UUID): args = payload.model_dump(exclude_none=True) - external_trace_id = get_external_trace_id(request) if external_trace_id: args["external_trace_id"] = external_trace_id - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error - except services.errors.conversation.ConversationCompletedError as caught_error: - raise ConversationCompletedError() from caught_error - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) from ex - except ValueError as e: - raise e - except Exception as caught_error: + result = application_services().console_workflows.generate(request_context, str(app_id), args) + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InternalServerError() from error @console_ns.route("/apps//advanced-chat/workflows/draft/iteration/nodes//run") @@ -788,40 +467,23 @@ class AdvancedChatDraftRunIterationNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(IterationNodeRunPayload) - def post(self, payload: IterationNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow iteration node - """ - args = payload.model_dump(exclude_none=True) - + def post(self, payload: IterationNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_iteration( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.iteration( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error - except services.errors.conversation.ConversationCompletedError as caught_error: - raise ConversationCompletedError() from caught_error - except ValueError as e: - raise e - except Exception as caught_error: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InternalServerError() from error @console_ns.route("/apps//workflows/draft/iteration/nodes//run") @@ -837,40 +499,23 @@ class WorkflowDraftRunIterationNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(IterationNodeRunPayload) - def post(self, payload: IterationNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow iteration node - """ - args = payload.model_dump(exclude_none=True) - + def post(self, payload: IterationNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_iteration( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.iteration( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error - except services.errors.conversation.ConversationCompletedError as caught_error: - raise ConversationCompletedError() from caught_error - except ValueError as e: - raise e - except Exception as caught_error: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InternalServerError() from error @console_ns.route("/apps//advanced-chat/workflows/draft/loop/nodes//run") @@ -882,39 +527,23 @@ class AdvancedChatDraftRunLoopNodeApi(Resource): @console_ns.response(200, "Loop node run started successfully", console_ns.models[GeneratedAppResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(LoopNodeRunPayload) - def post(self, args: LoopNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow loop node - """ - + def post(self, payload: LoopNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_loop( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.loop( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error - except services.errors.conversation.ConversationCompletedError as caught_error: - raise ConversationCompletedError() from caught_error - except ValueError as e: - raise e - except Exception as caught_error: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InternalServerError() from error @console_ns.route("/apps//workflows/draft/loop/nodes//run") @@ -930,39 +559,23 @@ class WorkflowDraftRunLoopNodeApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(LoopNodeRunPayload) - def post(self, args: LoopNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow loop node - """ - + def post(self, payload: LoopNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): try: - response = AppGenerateService.generate_single_loop( - app_model=app_model, - user=current_user, - node_id=node_id, - args=args, - session=db.session(), - streaming=True, + result = application_services().console_workflows.loop( + request_context, str(app_id), node_id, payload.inputs ) - - return helper.compact_generate_response(response) - except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error - except services.errors.conversation.ConversationCompletedError as caught_error: - raise ConversationCompletedError() from caught_error - except ValueError as e: - raise e - except Exception as caught_error: + return helper.compact_generate_response(result) + except services.errors.conversation.ConversationNotExistsError as error: + raise NotFoundError("Conversation Not Exists.") from error + except services.errors.conversation.ConversationCompletedError as error: + raise ConversationCompletedError() from error + except ValueError: + raise + except Exception as error: logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InternalServerError() from error class HumanInputFormPreviewPayload(BaseModel): @@ -1007,27 +620,11 @@ class AdvancedChatDraftHumanInputFormPreviewApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputFormPreviewPayload.__name__]) @console_ns.response(200, "Human input form preview", console_ns.models[HumanInputFormPreviewResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(HumanInputFormPreviewPayload) - def post(self, args: HumanInputFormPreviewPayload, current_user: Account, app_model: App, node_id: str): - """ - Preview human input form content and placeholders - """ - inputs = args.inputs - - workflow_service = WorkflowService() - preview = workflow_service.get_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - inputs=inputs, - session=db.session(), + def post(self, args: HumanInputFormPreviewPayload, request_context: RequestContext, app_id: UUID, node_id: str): + preview = application_services().human_input_debug.preview_form( + request_context, str(app_id), node_id, args.inputs ) return dump_response(HumanInputFormPreviewResponse, preview) @@ -1043,29 +640,21 @@ class AdvancedChatDraftHumanInputFormRunApi(Resource): "Human input form submission result", console_ns.models[HumanInputFormSubmitResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.ADVANCED_CHAT,)) @model_validate(HumanInputFormPreviewSubmitPayload) - def post(self, args: HumanInputFormPreviewSubmitPayload, current_user: Account, app_model: App, node_id: str): - """ - Submit human input form preview - """ - workflow_service = WorkflowService() - result = workflow_service.submit_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - form_inputs=args.form_inputs, - inputs=args.inputs, - action=args.action, - session=db.session(), + def post( + self, args: HumanInputFormPreviewSubmitPayload, request_context: RequestContext, app_id: UUID, node_id: str + ): + return jsonable_encoder( + application_services().human_input_debug.submit_form( + request_context, + str(app_id), + node_id, + inputs=args.inputs, + form_inputs=args.form_inputs, + action=args.action, + ) ) - return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/human-input/nodes//form/preview") @@ -1075,27 +664,11 @@ class WorkflowDraftHumanInputFormPreviewApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputFormPreviewPayload.__name__]) @console_ns.response(200, "Human input form preview", console_ns.models[HumanInputFormPreviewResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, modes=(AppMode.WORKFLOW,)) @model_validate(HumanInputFormPreviewPayload) - def post(self, args: HumanInputFormPreviewPayload, current_user: Account, app_model: App, node_id: str): - """ - Preview human input form content and placeholders - """ - inputs = args.inputs - - workflow_service = WorkflowService() - preview = workflow_service.get_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - inputs=inputs, - session=db.session(), + def post(self, args: HumanInputFormPreviewPayload, request_context: RequestContext, app_id: UUID, node_id: str): + preview = application_services().human_input_debug.preview_form( + request_context, str(app_id), node_id, args.inputs ) return dump_response(HumanInputFormPreviewResponse, preview) @@ -1111,29 +684,21 @@ class WorkflowDraftHumanInputFormRunApi(Resource): "Human input form submission result", console_ns.models[HumanInputFormSubmitResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(HumanInputFormPreviewSubmitPayload) - def post(self, args: HumanInputFormPreviewSubmitPayload, current_user: Account, app_model: App, node_id: str): - """ - Submit human input form preview - """ - workflow_service = WorkflowService() - result = workflow_service.submit_human_input_form_preview( - app_model=app_model, - account=current_user, - node_id=node_id, - form_inputs=args.form_inputs, - inputs=args.inputs, - action=args.action, - session=db.session(), + def post( + self, args: HumanInputFormPreviewSubmitPayload, request_context: RequestContext, app_id: UUID, node_id: str + ): + return jsonable_encoder( + application_services().human_input_debug.submit_form( + request_context, + str(app_id), + node_id, + inputs=args.inputs, + form_inputs=args.form_inputs, + action=args.action, + ) ) - return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/human-input/nodes//delivery-test") @@ -1143,28 +708,17 @@ class WorkflowDraftHumanInputDeliveryTestApi(Resource): @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @console_ns.expect(console_ns.models[HumanInputDeliveryTestPayload.__name__]) @console_ns.response(200, "Human input delivery test result", console_ns.models[EmptyObjectResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) @model_validate(HumanInputDeliveryTestPayload) - def post(self, args: HumanInputDeliveryTestPayload, current_user: Account, app_model: App, node_id: str): - """ - Test human input delivery - """ - workflow_service = WorkflowService() - workflow_service.test_human_input_delivery( - app_model=app_model, - account=current_user, - node_id=node_id, - delivery_method_id=args.delivery_method_id, + def post(self, args: HumanInputDeliveryTestPayload, request_context: RequestContext, app_id: UUID, node_id: str): + application_services().human_input_debug.test_delivery( + request_context, + str(app_id), + node_id, inputs=args.inputs, - session=db.session(), + delivery_method_id=args.delivery_method_id, ) - return jsonable_encoder({}) + return {} @console_ns.route("/apps//workflows/draft/run") @@ -1180,38 +734,18 @@ class DraftWorkflowRunApi(Resource): console_ns.models[GeneratedAppResponse.__name__], ) @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowRunPayload) - def post(self, payload: DraftWorkflowRunPayload, session: Session, current_user: Account, app_model: App): - """ - Run draft workflow - """ + def post(self, payload: DraftWorkflowRunPayload, request_context: RequestContext, app_id: UUID): args = payload.model_dump(exclude_none=True) - external_trace_id = get_external_trace_id(request) if external_trace_id: args["external_trace_id"] = external_trace_id - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - ) - - return helper.compact_generate_response(response) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) from ex + result = application_services().console_workflows.generate(request_context, str(app_id), args) + return helper.compact_generate_response(result) + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error @console_ns.route("/apps//workflow-runs/tasks//stop") @@ -1222,24 +756,9 @@ class WorkflowTaskStopApi(Resource): @console_ns.response(200, "Task stopped successfully", console_ns.models[SimpleResultResponse.__name__]) @console_ns.response(404, "Task not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def post(self, app_model: App, task_id: str): - """ - Stop workflow task - """ - AppTaskService.stop_workflow_task( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - task_id=task_id, - app_mode=AppMode.value_of(app_model.mode), - owner=None, - ) - + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) + def post(self, request_context: RequestContext, app_id: UUID, task_id: str): + application_services().console_workflows.stop(request_context, str(app_id), task_id) return {"result": "success"} @@ -1256,46 +775,19 @@ class DraftWorkflowNodeRunApi(Resource): ) @console_ns.response(403, "Permission denied") @console_ns.response(404, "Node not found") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN) @model_validate(DraftWorkflowNodeRunPayload) - def post(self, payload: DraftWorkflowNodeRunPayload, current_user: Account, app_model: App, node_id: str): - """ - Run draft workflow node - """ - args = payload.model_dump(exclude_none=True) - - user_inputs = payload.inputs - if user_inputs is None: - raise ValueError("missing inputs") - - workflow_srv = WorkflowService() - # fetch draft workflow by app_model - draft_workflow = workflow_srv.get_draft_workflow(app_model=app_model, session=db.session()) - if not draft_workflow: - raise ValueError("Workflow not initialized") - files = _parse_file(draft_workflow, args.get("files")) - workflow_service = WorkflowService() - - workflow_node_execution = workflow_service.run_draft_workflow_node( - app_model=app_model, - draft_workflow=draft_workflow, - node_id=node_id, - user_inputs=user_inputs, - account=current_user, - query=args.get("query", ""), - files=files, + def post(self, payload: DraftWorkflowNodeRunPayload, request_context: RequestContext, app_id: UUID, node_id: str): + return dump_response( + WorkflowRunNodeExecutionResponse, + application_services().console_workflows.run_node( + request_context, + str(app_id), + node_id, + payload.model_dump(exclude_none=True), + ), ) - return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True - ).model_dump(mode="json") - @console_ns.route("/apps//workflows/publish") class PublishedWorkflowApi(Resource): @@ -1307,66 +799,25 @@ class PublishedWorkflowApi(Resource): "Published workflow retrieved successfully, or null if not found", console_ns.models[WorkflowResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App): - """ - Get published workflow - """ - # fetch published workflow by app_model - workflow_service = WorkflowService() - session = db.session() - workflow = workflow_service.get_published_workflow(app_model=app_model, session=session) - - # return workflow, if not found, return None - if workflow is None: - return None - - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + result = application_services().console_workflows.published(request_context, str(app_id)) + return dump_response(WorkflowResponse, result) if result is not None else None @console_ns.expect(console_ns.models[PublishWorkflowPayload.__name__]) @console_ns.response(200, "Workflow published successfully", console_ns.models[WorkflowPublishResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_RELEASE_AND_VERSION, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required + @console_workflow_admission(permission=RBACPermission.APP_RELEASE_AND_VERSION) @model_validate(PublishWorkflowPayload) - def post(self, args: PublishWorkflowPayload, current_user: Account, app_model: App): - """ - Publish workflow - """ - - workflow_service = WorkflowService() - with sessionmaker(db.engine).begin() as session: - workflow = workflow_service.publish_workflow( - session=session, - app_model=app_model, - account=current_user, - marked_name=args.marked_name or "", - marked_comment=args.marked_comment or "", - ) - - # Update app_model within the same session to ensure atomicity - app_model_in_session = session.get(App, app_model.id) - if app_model_in_session: - app_model_in_session.workflow_id = workflow.id - app_model_in_session.updated_by = current_user.id - app_model_in_session.updated_at = naive_utc_now() - - workflow_created_at = TimestampField().format(workflow.created_at) - graph_text = workflow.graph - - warning = advisory_variable_reference_warning(graph_text) + def post(self, args: PublishWorkflowPayload, request_context: RequestContext, app_id: UUID): + publication, warning = application_services().console_workflows.publish( + request_context, + str(app_id), + marked_name=args.marked_name or "", + marked_comment=args.marked_comment or "", + ) payload: dict[str, object] = { "result": "success", - "created_at": workflow_created_at, + "created_at": TimestampField().format(publication.created_at), } if warning: payload["warning"] = warning @@ -1383,19 +834,9 @@ class DefaultBlockConfigsApi(Resource): "Default block configurations retrieved successfully", console_ns.models[DefaultBlockConfigsResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App): - """ - Get default block config - """ - # Get default block configs - workflow_service = WorkflowService() - return workflow_service.get_default_block_configs() + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) + def get(self, request_context: RequestContext, app_id: UUID): + return application_services().console_workflows.default_blocks() @console_ns.route("/apps//workflows/default-workflow-block-configs/") @@ -1410,28 +851,16 @@ class DefaultBlockConfigApi(Resource): ) @console_ns.response(404, "Block type not found") @console_ns.doc(params=query_params_from_model(DefaultBlockConfigQuery)) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(DefaultBlockConfigQuery) - def get(self, args: DefaultBlockConfigQuery, app_model: App, block_type: str): - """ - Get default block config - """ - + def get(self, args: DefaultBlockConfigQuery, request_context: RequestContext, app_id: UUID, block_type: str): filters = None if args.q: try: filters = json.loads(args.q) - except json.JSONDecodeError as caught_error: - raise ValueError("Invalid filters") from caught_error - - # Get default block configs - workflow_service = WorkflowService() - return workflow_service.get_default_block_config(node_type=block_type, filters=filters) + except json.JSONDecodeError as error: + raise ValueError("Invalid filters") from error + return application_services().console_workflows.default_block(block_type, filters) @console_ns.route("/apps//convert-to-workflow") @@ -1447,40 +876,16 @@ class ConvertToWorkflowApi(Resource): ) @console_ns.response(400, "Application cannot be converted") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.CHAT, AppMode.COMPLETION]) - @with_current_user - @with_current_tenant_id - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_EDIT, modes=(AppMode.CHAT, AppMode.COMPLETION)) @model_validate(ConvertToWorkflowPayload) - def post(self, payload: ConvertToWorkflowPayload, current_tenant_id: str, current_user: Account, app_model: App): - """ - Convert basic mode of chatbot app to workflow mode - Convert expert mode of chatbot app to workflow mode - Convert Completion App to Workflow App - """ - args = payload.model_dump(exclude_none=True) - - # convert to workflow mode - workflow_service = WorkflowService() - new_app_model = workflow_service.convert_to_workflow( - app_model=app_model, account=current_user, args=args, session=db.session() + def post(self, payload: ConvertToWorkflowPayload, request_context: RequestContext, app_id: UUID): + return application_services().console_workflows.convert( + request_context, str(app_id), payload.model_dump(exclude_none=True) ) - # return app id - return { - "new_app_id": new_app_model.id, - "permission_keys": get_app_permission_keys(str(current_tenant_id), current_user.id, str(new_app_model.id)), - } - @console_ns.route("/apps//workflows/draft/features") class WorkflowFeaturesApi(Resource): - """Update draft workflow features.""" - @console_ns.expect(console_ns.models[WorkflowFeaturesPayload.__name__]) @console_ns.doc("update_workflow_features") @console_ns.doc(description="Update draft workflow features") @@ -1490,22 +895,14 @@ class WorkflowFeaturesApi(Resource): "Workflow features updated successfully", console_ns.models[SimpleResultResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(WorkflowFeaturesPayload) - def post(self, args: WorkflowFeaturesPayload, current_user: Account, app_model: App): - features = args.features.model_dump(mode="json", exclude_unset=True) - - workflow_service = WorkflowService() - workflow_service.update_draft_workflow_features( - app_model=app_model, features=features, account=current_user, session=db.session() + def post(self, args: WorkflowFeaturesPayload, request_context: RequestContext, app_id: UUID): + application_services().console_workflows.update_features( + request_context, + str(app_id), + args.features.model_dump(mode="json", exclude_unset=True), ) - return {"result": "success"} @@ -1520,46 +917,20 @@ class PublishedAllWorkflowApi(Resource): "Published workflows retrieved successfully", console_ns.models[WorkflowPaginationResponse.__name__], ) - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @with_current_user - @edit_permission_required - @with_session(write=False) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT) @model_validate(WorkflowListQuery) - def get(self, args: WorkflowListQuery, session: Session, current_user: Account, app_model: App): - """ - Get published workflows - """ - - page = args.page - limit = args.limit - user_id = args.user_id - named_only = args.named_only - - if user_id: - if user_id != current_user.id: - raise Forbidden() - - workflow_service = WorkflowService() - workflows, has_more = workflow_service.get_all_published_workflow( - session=session, - app_model=app_model, - page=page, - limit=limit, - user_id=user_id, - named_only=named_only, + def get(self, args: WorkflowListQuery, request_context: RequestContext, app_id: UUID): + items, has_more = application_services().console_workflows.versions( + request_context, + str(app_id), + page=args.page, + limit=args.limit, + user_id=args.user_id, + named_only=args.named_only, + ) + return dump_response( + WorkflowPaginationResponse, {"items": items, "page": args.page, "limit": args.limit, "has_more": has_more} ) - return WorkflowPaginationResponse.model_validate( - { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], - "page": page, - "limit": limit, - "has_more": has_more, - } - ).model_dump(mode="json") @console_ns.route("/apps//workflows//restore") @@ -1570,35 +941,17 @@ class DraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Workflow restored successfully", console_ns.models[WorkflowRestoreResponse.__name__]) @console_ns.response(400, "Source workflow must be published") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_RELEASE_AND_VERSION, PlainApp())) - def post(self, current_user: Account, app_model: App, workflow_id: str): - workflow_service = WorkflowService() - + @console_workflow_admission(permission=RBACPermission.APP_RELEASE_AND_VERSION) + def post(self, request_context: RequestContext, app_id: UUID, workflow_id: str): try: - workflow = workflow_service.restore_published_workflow_to_draft( - app_model=app_model, - workflow_id=workflow_id, - account=current_user, - session=db.session(), - ) - except IsDraftWorkflowError as exc: - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc - except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc - except ValueError as exc: - raise BadRequest(str(exc)) from exc - - return { - "result": "success", - "hash": workflow.unique_hash, - "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), - } + result = application_services().console_workflows.restore(request_context, str(app_id), workflow_id) + except IsDraftWorkflowError as error: + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + except ValueError as error: + raise InvalidRequestError(str(error)) from error + return {"result": "success", "hash": result.hash, "updated_at": TimestampField().format(result.updated_at)} @console_ns.route("/apps//workflows/") @@ -1610,82 +963,24 @@ class WorkflowByIdApi(Resource): @console_ns.response(200, "Workflow updated successfully", console_ns.models[WorkflowResponse.__name__]) @console_ns.response(404, "Workflow not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) + @console_workflow_admission(permission=RBACPermission.APP_EDIT) @model_validate(WorkflowUpdatePayload) - def patch(self, args: WorkflowUpdatePayload, current_user: Account, app_model: App, workflow_id: str): - """ - Update workflow attributes - """ - - # Prepare update data - update_data = {} - if args.marked_name is not None: - update_data["marked_name"] = args.marked_name - if args.marked_comment is not None: - update_data["marked_comment"] = args.marked_comment - - if not update_data: + def patch(self, args: WorkflowUpdatePayload, request_context: RequestContext, app_id: UUID, workflow_id: str): + changes = args.model_dump(exclude_none=True) + if not changes: raise InvalidArgumentError("No valid fields to update") + return dump_response( + WorkflowResponse, + application_services().console_workflows.update(request_context, str(app_id), workflow_id, changes), + ) - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id) - - # Create a session and manage the transaction - with sessionmaker(db.engine, expire_on_commit=False).begin() as session: - workflow = workflow_service.update_workflow( - session=session, - account_id=current_user.id, - data=update_data, - workflow_ref=workflow_ref, - ) - - if not workflow: - raise NotFound("Workflow not found") - - response = dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) - - return response - - @setup_required - @login_required - @account_initialization_required - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_EDIT, PlainApp())) @console_ns.response(204, "Workflow deleted successfully") - def delete(self, current_user: Account, app_model: App, workflow_id: str): - """ - Delete workflow - """ - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id) - - # Create a session and manage the transaction - with sessionmaker(db.engine).begin() as session: - try: - retirement_candidates = workflow_service.delete_workflow( - session=session, - workflow_ref=workflow_ref, - ) - except WorkflowInUseError as e: - abort(400, description=str(e)) - except DraftWorkflowDeletionError as e: - abort(400, description=str(e)) - except ValueError as e: - raise NotFound(str(e)) from e - - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=current_user.id, - ) + @console_workflow_admission(permission=RBACPermission.APP_EDIT) + def delete(self, request_context: RequestContext, app_id: UUID, workflow_id: str): + try: + application_services().console_workflows.delete(request_context, WorkflowOwner(str(app_id)), workflow_id) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) from error return None, 204 @@ -1701,26 +996,12 @@ class DraftWorkflowNodeLastRunApi(Resource): ) @console_ns.response(404, "Node last run not found") @console_ns.response(403, "Permission denied") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_VIEW_LAYOUT, PlainApp())) - @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - def get(self, app_model: App, node_id: str): - srv = WorkflowService() - workflow = srv.get_draft_workflow(app_model, session=db.session()) - if not workflow: - raise NotFound("Workflow not found") - node_exec = srv.get_node_last_run( - app_model=app_model, - workflow=workflow, - node_id=node_id, + @console_workflow_admission(permission=RBACPermission.APP_VIEW_LAYOUT, require_editor=False) + def get(self, request_context: RequestContext, app_id: UUID, node_id: str): + return dump_response( + WorkflowRunNodeExecutionResponse, + application_services().console_workflows.last_run(request_context, str(app_id), node_id), ) - if node_exec is None: - raise NotFound("last run not found") - return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=db.session()), from_attributes=True - ).model_dump(mode="json") @console_ns.route("/apps//workflows/draft/trigger/run") @@ -1743,64 +1024,27 @@ class DraftWorkflowTriggerRunApi(Resource): @console_ns.response(400, "Trigger run failed", console_ns.models[WorkflowTriggerRunErrorResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_current_user - @edit_permission_required - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowTriggerRunPayload) - def post(self, args: DraftWorkflowTriggerRunPayload, session: Session, current_user: Account, app_model: App): - """ - Poll for trigger events and execute full workflow when event arrives - """ - node_id = args.node_id - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not found") - - poller: TriggerDebugEventPoller = create_event_poller( - draft_workflow=draft_workflow, - tenant_id=app_model.tenant_id, - user_id=current_user.id, - app_id=app_model.id, - node_id=node_id, - ) - event: TriggerDebugEvent | None = None + def post(self, args: DraftWorkflowTriggerRunPayload, request_context: RequestContext, app_id: UUID): try: - event = poller.poll() - if not event: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - return helper.compact_generate_response( - AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=event.workflow_args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - root_node_id=node_id, - ) + result = application_services().console_workflows.trigger( + request_context, str(app_id), [args.node_id], single_node=False, select_all=False ) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) from ex - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return helper.compact_generate_response(result) @console_ns.route("/apps//workflows/draft/nodes//trigger/run") class DraftWorkflowTriggerNodeApi(Resource): - """ - Single node debug - Polling API for trigger events - Path: /apps//workflows/draft/nodes//trigger/run - """ - @console_ns.doc("poll_draft_workflow_trigger_node") @console_ns.doc(description="Poll for trigger events and execute single node when event arrives") @console_ns.doc(params={"app_id": "Application ID", "node_id": "Node ID"}) @@ -1812,71 +1056,22 @@ class DraftWorkflowTriggerNodeApi(Resource): @console_ns.response(400, "Trigger run failed", console_ns.models[WorkflowTriggerRunErrorResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @get_app_model(mode=[AppMode.WORKFLOW]) - @with_current_user - @edit_permission_required - def post(self, current_user: Account, app_model: App, node_id: str): - """ - Poll for trigger events and execute single node when event arrives - """ - - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=db.session()) - if not draft_workflow: - raise ValueError("Workflow not found") - - node_config = draft_workflow.get_node_config_by_id(node_id=node_id) - if not node_config: - raise ValueError("Node data not found for node %s", node_id) - node_type: NodeType = draft_workflow.get_node_type_from_node_config(node_config) - event: TriggerDebugEvent | None = None - # for schedule trigger, when run single node, just execute directly - if node_type == TRIGGER_SCHEDULE_NODE_TYPE: - event = TriggerDebugEvent( - workflow_args={}, - node_id=node_id, - ) - # for other trigger types, poll for the event - else: - try: - poller: TriggerDebugEventPoller = create_event_poller( - draft_workflow=draft_workflow, - tenant_id=app_model.tenant_id, - user_id=current_user.id, - app_id=app_model.id, - node_id=node_id, - ) - event = poller.poll() - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e - if not event: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - - raw_files = event.workflow_args.get("files") - files = _parse_file(draft_workflow, raw_files if isinstance(raw_files, list) else None) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) + def post(self, request_context: RequestContext, app_id: UUID, node_id: str): try: - node_execution = workflow_service.run_draft_workflow_node( - app_model=app_model, - draft_workflow=draft_workflow, - node_id=node_id, - user_inputs=event.workflow_args.get("inputs") or {}, - account=current_user, - query="", - files=files, + result = application_services().console_workflows.trigger( + request_context, str(app_id), [node_id], single_node=True, select_all=False ) - return jsonable_encoder(node_execution) - except Exception: - logger.exception("Error running draft workflow trigger node") - return jsonable_encoder( - {"status": "error", "error": "An unexpected error occurred while running the node."} - ), 400 + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return jsonable_encoder(result) @console_ns.route("/apps//workflows/draft/trigger/run-all") @@ -1895,61 +1090,23 @@ class DraftWorkflowTriggerRunAllApi(Resource): @console_ns.response(400, "Trigger run failed", console_ns.models[WorkflowTriggerRunErrorResponse.__name__]) @console_ns.response(403, "Permission denied") @console_ns.response(500, "Internal server error") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.APP_TEST_AND_RUN, PlainApp())) - @with_session - @get_app_model(mode=[AppMode.WORKFLOW]) + @console_workflow_admission(permission=RBACPermission.APP_TEST_AND_RUN, modes=(AppMode.WORKFLOW,)) @model_validate(DraftWorkflowTriggerRunAllPayload) - def post(self, args: DraftWorkflowTriggerRunAllPayload, session: Session, current_user: Account, app_model: App): - """ - Full workflow debug when the start node is a trigger - """ - - node_ids = args.node_ids - workflow_service = WorkflowService() - draft_workflow = workflow_service.get_draft_workflow(app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not found") - + def post(self, args: DraftWorkflowTriggerRunAllPayload, request_context: RequestContext, app_id: UUID): try: - trigger_debug_event: TriggerDebugEvent | None = select_trigger_debug_events( - draft_workflow=draft_workflow, - app_model=app_model, - user_id=current_user.id, - node_ids=node_ids, + result = application_services().console_workflows.trigger( + request_context, str(app_id), args.node_ids, single_node=False, select_all=True ) - except PluginInvokeError as e: - return jsonable_encoder({"status": "error", "error": e.to_user_friendly_error()}), 400 - except Exception as e: - logger.exception("Error polling trigger debug event") - raise e - if trigger_debug_event is None: - return jsonable_encoder({"status": "waiting", "retry_in": LISTENING_RETRY_IN}) - - try: - response = AppGenerateService.generate( - session=session, - app_model=app_model, - user=current_user, - args=trigger_debug_event.workflow_args, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - root_node_id=trigger_debug_event.node_id, - ) - return helper.compact_generate_response(response) - except InvokeRateLimitError as ex: - raise InvokeRateLimitHttpError(ex.description) from ex - except Exception: - logger.exception("Error running draft workflow trigger run-all") - return jsonable_encoder( - { - "status": "error", - } - ), 400 + except WorkflowTriggerError as error: + payload = {"status": "error"} + if error.message is not None: + payload["error"] = error.message + return payload, 400 + except InvokeRateLimitError as error: + raise InvokeRateLimitHttpError(error.description) from error + if result is None: + return {"status": "waiting", "retry_in": LISTENING_RETRY_IN} + return helper.compact_generate_response(result) @console_ns.route("/apps/workflows/online-users") @@ -1962,85 +1119,11 @@ class WorkflowOnlineUsersApi(Resource): ) @console_ns.doc("get_workflow_online_users") @console_ns.doc(description="Get workflow online users") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @with_current_tenant_id + @console_account_admission() @model_validate(WorkflowOnlineUsersPayload) - def post(self, args: WorkflowOnlineUsersPayload, current_tenant_id: str, current_user: Account): - app_ids = args.app_ids - if len(app_ids) > MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS: - raise BadRequest(f"Maximum {MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS} app_ids are allowed per request.") - - if not app_ids: - return {"data": []} - - access_filter = None - if dify_config.RBAC_ENABLED: - access_filter = resolve_app_access_filter(current_tenant_id, current_user.id) - workflow_service = WorkflowService() - with session_factory.create_session() as session: - app_maintainers = workflow_service.get_tenant_app_maintainers(app_ids, current_tenant_id, session=session) - - accessible_app_ids = set(app_maintainers) - if access_filter is not None: - accessible_app_ids = { - app_id - for app_id, maintainer in app_maintainers.items() - if access_filter.is_app_accessible(app_id, maintainer, current_user.id) - } - ordered_accessible_app_ids = [app_id for app_id in app_ids if app_id in accessible_app_ids] - - users_json_by_app_id: dict[str, Any] = {} - for start_index in range(0, len(ordered_accessible_app_ids), WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE): - app_id_batch = ordered_accessible_app_ids[ - start_index : start_index + WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE - ] - pipe = redis_client.pipeline(transaction=False) - for app_id in app_id_batch: - pipe.hgetall(f"{WORKFLOW_ONLINE_USERS_PREFIX}{app_id}") - - users_json_batch = pipe.execute() - for app_id, users_json in zip(app_id_batch, users_json_batch, strict=False): - users_json_by_app_id[app_id] = users_json - - results = [] - for app_id in ordered_accessible_app_ids: - users_json = users_json_by_app_id.get(app_id, {}) - - users = [] - for _, user_info_json in users_json.items(): - try: - user_info = json.loads(user_info_json) - except Exception: - continue - - if not isinstance(user_info, dict): - continue - - user_id = user_info.get("user_id") - username = user_info.get("username") - if not isinstance(user_id, str) or not isinstance(username, str): - continue - - avatar = user_info.get("avatar") - if avatar is not None and not isinstance(avatar, str): - avatar = None - - if isinstance(avatar, str) and avatar and not avatar.startswith(("http://", "https://")): - try: - avatar = file_helpers.get_signed_file_url(avatar) - except Exception as exc: - logger.warning( - "Failed to sign workflow online user avatar; using original value. " - "app_id=%s avatar=%s error=%s", - app_id, - avatar, - exc, - ) - - users.append({"user_id": user_id, "username": username, "avatar": avatar}) - results.append({"app_id": app_id, "users": users}) - - return WorkflowOnlineUsersResponse.model_validate({"data": results}).model_dump(mode="json") + def post(self, args: WorkflowOnlineUsersPayload, request_context: RequestContext): + try: + data = application_services().console_workflows.online_users(request_context, args.app_ids) + except ValueError as error: + raise InvalidRequestError(str(error)) from error + return dump_response(WorkflowOnlineUsersResponse, {"data": data}) diff --git a/api/controllers/console/app/workflow_admission.py b/api/controllers/console/app/workflow_admission.py new file mode 100644 index 00000000000000..efa158729cfbaf --- /dev/null +++ b/api/controllers/console/app/workflow_admission.py @@ -0,0 +1,58 @@ +"""Console workflow admission and domain-to-HTTP error translation.""" + +from collections.abc import Callable +from functools import wraps +from typing import Concatenate + +from controllers.common.errors import AccessDeniedError, NotFoundError +from controllers.common.rbac import PlainApp, RBACCheck +from controllers.console.app.error import AppNotFoundError, DraftWorkflowNotExist, DraftWorkflowNotSync +from controllers.console.flask_admission import console_account_admission +from controllers.console.wraps import RBACPermission +from extensions.ext_application_services import application_services +from machinery.context import RequestContext +from models.account import TenantAccountRole +from models.model import AppMode +from services.app.console_service import ConsoleAppNotFoundError +from services.errors.app import WorkflowHashNotEqualError, WorkflowNotFoundError +from services.errors.base import NoPermissionError +from services.workflow.contracts import DraftWorkflowMissingError + +_EDIT_ROLES = frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}) + + +def console_workflow_admission[T, **P, R]( + *, + permission: RBACPermission, + modes: tuple[AppMode, ...] = (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW), + require_editor: bool = True, +): + def decorator(view: Callable[Concatenate[T, RequestContext, P], R]): + @console_account_admission( + allowed_roles=_EDIT_ROLES if require_editor else None, + rbac_checks=[RBACCheck(permission, PlainApp())], + ) + @wraps(view) + def admitted(self: T, context: RequestContext, /, *args: P.args, **kwargs: P.kwargs) -> R: + app_id = kwargs.get("app_id") + if app_id is None: + raise ValueError("missing app_id in path parameters") + try: + app = application_services().apps.console.get_reference(context, str(app_id)) + if app.mode not in modes: + raise AppNotFoundError(f"App mode is not in the supported list: {set(modes)}") + return view(self, context, *args, **kwargs) + except ConsoleAppNotFoundError as error: + raise AppNotFoundError() from error + except DraftWorkflowMissingError as error: + raise DraftWorkflowNotExist() from error + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + except NoPermissionError as error: + raise AccessDeniedError() from error + + return admitted + + return decorator diff --git a/api/controllers/console/app/workflow_node_output_inspector.py b/api/controllers/console/app/workflow_node_output_inspector.py index f473d0313750ba..7ac584e510000b 100644 --- a/api/controllers/console/app/workflow_node_output_inspector.py +++ b/api/controllers/console/app/workflow_node_output_inspector.py @@ -41,6 +41,7 @@ rbac_permission_required, setup_required, ) +from extensions.ext_application_services import application_services from extensions.ext_database import db from libs.exception import BaseHTTPException, RelayedHTTPError from libs.external_api import error_responses @@ -82,7 +83,7 @@ def _service() -> NodeOutputInspectorService: """One-line factory so tests can monkeypatch a stub if needed.""" - return NodeOutputInspectorService() + return NodeOutputInspectorService(binding_resolver=application_services().workflow_agent_bindings) def _serve_snapshot(app_model: App, run_id: UUID) -> dict: diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py index 048f5cf858db96..efdd6ebfa503f6 100644 --- a/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py +++ b/api/controllers/console/datasets/rag_pipeline/datasource_content_preview.py @@ -10,11 +10,9 @@ from controllers.console.datasets.wraps import get_rag_pipeline from controllers.console.wraps import account_initialization_required, model_validate, setup_required, with_current_user from extensions.ext_application_services import application_services -from extensions.ext_database import db from libs.login import login_required from models import Account from models.dataset import Pipeline -from services.rag_pipeline.rag_pipeline import RagPipelineService class DatasourceContentPreviewPayload(BaseModel): @@ -43,8 +41,7 @@ def post(self, req_data: DatasourceContentPreviewPayload, current_user: Account, inputs = req_data.inputs datasource_type = req_data.datasource_type - rag_pipeline_service = RagPipelineService(db.session()) - preview_content = rag_pipeline_service.run_datasource_node_preview( + preview_content = application_services().knowledge.pipeline_execution.run_datasource_node_preview( pipeline=pipeline, node_id=node_id, user_inputs=inputs, diff --git a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py index 24eb44a893d307..331470cc1453b7 100644 --- a/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py +++ b/api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py @@ -3,16 +3,22 @@ from typing import Any, Literal, cast from uuid import UUID -from flask import abort, request +from flask import request from flask_restx import Resource from pydantic import BaseModel, Field, RootModel from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound import services.errors.base from configs import dify_config from controllers.common.controller_schemas import DefaultBlockConfigQuery, WorkflowListQuery, WorkflowUpdatePayload -from controllers.common.errors import InvalidArgumentError +from controllers.common.errors import ( + AccessDeniedError, + InternalServerError, + InvalidArgumentError, + InvalidRequestError, + NotFoundError, + UnsupportedMediaTypeError, +) from controllers.common.fields import SimpleResultResponse from controllers.common.rbac import DatasetByPipeline, DatasetId, RBACCheck from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models @@ -22,18 +28,10 @@ DraftWorkflowNotExist, DraftWorkflowNotSync, ) -from controllers.console.app.workflow import ( - RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, - DefaultBlockConfigResponse, - DefaultBlockConfigsResponse, - PipelineVariableResponse, - WorkflowPaginationResponse, - WorkflowResponse, - WorkflowResponseSource, -) from controllers.console.app.wraps import with_session from controllers.console.datasets.error import PipelineNotFoundError from controllers.console.datasets.wraps import get_rag_pipeline, load_rag_pipeline +from controllers.console.flask_admission import console_account_admission from controllers.console.workspace.tool_providers import ToolProviderApiEntityResponse from controllers.console.wraps import ( RBACPermission, @@ -47,15 +45,21 @@ ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError from core.app.apps.base_app_queue_manager import AppQueueManager -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.task_entities import WorkflowAppBlockingResponse from core.entities.knowledge_entities import PipelineGenerateResponse from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity +from enums.account import TenantAccountRole from extensions.ext_application_services import application_services from extensions.ext_database import db -from factories import variable_factory from fields.base import ResponseModel +from fields.workflow_fields import ( + DefaultBlockConfigResponse, + DefaultBlockConfigsResponse, + PipelineVariableResponse, + WorkflowPaginationResponse, + WorkflowResponse, +) from fields.workflow_run_fields import ( WorkflowRunDetailResponse, WorkflowRunNodeExecutionListResponse, @@ -69,21 +73,26 @@ from libs.external_api import error_responses from libs.helper import TimestampField, UUIDStrOrEmpty, dump_response from libs.login import login_required +from machinery.context import RequestContext from models import Account from models.dataset import Pipeline from models.model import EndUser -from models.workflow import Workflow -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.app.generation.response import convert_to_event_stream from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.llm import InvokeRateLimitError -from services.errors.rag_pipeline import RagPipelineResourceNotFoundError +from services.errors.rag_pipeline import RagPipelinePublicationError, RagPipelineResourceNotFoundError +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.knowledge.dataset_service import DatasetService from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService from services.rag_pipeline.rag_pipeline import RagPipelineService from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService from services.rag_pipeline.rag_pipeline_transform_service import RagPipelineTransformService +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, +) from services.workflow_ref_service import WorkflowRefService -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService logger = logging.getLogger(__name__) @@ -251,25 +260,22 @@ def get(self, session: Session, pipeline: Pipeline): """ # fetch draft workflow by app_model rag_pipeline_service = RagPipelineService(session) - workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) + workflow = rag_pipeline_service.get_draft_workflow_record(pipeline=pipeline) if not workflow: raise DraftWorkflowNotExist() # return workflow, if not found, return 404 - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(WorkflowResponse, workflow) @error_responses(DraftWorkflowNotSync) - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_rag_pipeline - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) @console_ns.expect(console_ns.models[DraftWorkflowSyncPayload.__name__]) @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowSyncResponse.__name__]) - def post(self, current_user: Account, pipeline: Pipeline): + def post(self, context: RequestContext, pipeline_id: UUID): """ Sync draft workflow """ @@ -284,35 +290,32 @@ def post(self, current_user: Account, pipeline: Pipeline): except ValueError as caught_error: raise InvalidArgumentError("Invalid JSON data") from caught_error else: - abort(415) - rag_pipeline_service = RagPipelineService(db.session()) + raise UnsupportedMediaTypeError() try: - environment_variables_list = Workflow.normalize_environment_variable_mappings( - payload.environment_variables or [], - ) - environment_variables = [ - variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list - ] - conversation_variables_list = payload.conversation_variables or [] - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list - ] - workflow = rag_pipeline_service.sync_draft_workflow( - pipeline=pipeline, - graph=payload.graph, - unique_hash=payload.hash, - account=current_user, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - rag_pipeline_variables=payload.rag_pipeline_variables or [], + workflow = application_services().workflow_drafts.sync( + context, + WorkflowOwner(str(pipeline_id), "pipeline"), + DraftSyncCommand( + graph=payload.graph, + features={}, + unique_hash=payload.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=payload.environment_variables or [], + conversation_variables=payload.conversation_variables or [], + rag_pipeline_variables=payload.rag_pipeline_variables or [], + ), ) + except WorkflowNotFoundError as exc: + raise NotFoundError(str(exc)) from exc except WorkflowHashNotEqualError as caught_error: raise DraftWorkflowNotSync() from caught_error return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -342,13 +345,12 @@ def post(self, req_data: NodeRunPayload, current_user: Account, pipeline: Pipeli user=current_user, node_id=node_id, args=args, - session=db.session(), streaming=True, ) return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except ValueError as e: @@ -383,13 +385,12 @@ def post(self, req_data: NodeRunPayload, current_user: Account, pipeline: Pipeli user=current_user, node_id=node_id, args=args, - session=db.session(), streaming=True, ) return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except ValueError as e: @@ -423,8 +424,8 @@ def post( try: response = PipelineGenerateService.generate( - generator=application_services().knowledge.pipeline_generator, session=session, + generator=application_services().knowledge.pipeline_generator, pipeline=pipeline, user=current_user, args=args, @@ -460,8 +461,8 @@ def post(self, req_data: PublishedWorkflowRunPayload, session: Session, current_ try: response = PipelineGenerateService.generate( - generator=application_services().knowledge.pipeline_generator, session=session, + generator=application_services().knowledge.pipeline_generator, pipeline=pipeline, user=current_user, args=args, @@ -491,10 +492,9 @@ def post(self, req_data: DatasourceNodeRunPayload, current_user: Account, pipeli Run rag pipeline datasource """ - rag_pipeline_service = RagPipelineService(db.session()) return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=req_data.inputs, @@ -525,10 +525,9 @@ def post(self, req_data: DatasourceNodeRunPayload, current_user: Account, pipeli Run rag pipeline datasource """ - rag_pipeline_service = RagPipelineService(db.session()) return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=req_data.inputs, @@ -564,8 +563,7 @@ def post(self, req_data: NodeRunRequiredPayload, current_user: Account, pipeline """ inputs = req_data.inputs - rag_pipeline_service = RagPipelineService(db.session()) - workflow_node_execution = rag_pipeline_service.run_draft_workflow_node( + workflow_node_execution = application_services().knowledge.pipeline_execution.run_draft_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=inputs, account=current_user ) @@ -573,7 +571,12 @@ def post(self, req_data: NodeRunRequiredPayload, current_user: Account, pipeline raise ValueError("Workflow node execution not found") return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True + node_execution_response_source( + workflow_node_execution, + session=db.session(), + tool_providers=application_services().tools.tool_providers, + ), + from_attributes=True, ).model_dump(mode="json") @@ -619,41 +622,30 @@ def get(self, pipeline: Pipeline): # fetch published workflow by pipeline session = db.session() rag_pipeline_service = RagPipelineService(session) - workflow = rag_pipeline_service.get_published_workflow(pipeline=pipeline) + workflow = rag_pipeline_service.get_published_workflow_record(pipeline=pipeline) # return workflow, if not found, return None if workflow is None: return None - return dump_response(RagPipelinePublishedWorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(RagPipelinePublishedWorkflowResponse, workflow) @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowPublishResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def post(self, current_user: Account, pipeline: Pipeline): - """ - Publish workflow - """ - rag_pipeline_service = RagPipelineService(db.session()) - workflow = rag_pipeline_service.publish_workflow( - session=db.session(), - pipeline=pipeline, - account=current_user, - ) - pipeline.is_published = True - pipeline.workflow_id = workflow.id - db.session.commit() - workflow_created_at = TimestampField().format(workflow.created_at) - - return { - "result": "success", - "created_at": workflow_created_at, - } + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def post(self, context: RequestContext, pipeline_id: str): + """Publish a prepared Pipeline and its dataset settings atomically.""" + try: + workflow = application_services().knowledge.pipeline_publication.publish(context, str(pipeline_id)) + except RagPipelinePublicationError as error: + raise InvalidRequestError(str(error)) from error + except WorkflowHashNotEqualError as error: + raise DraftWorkflowNotSync() from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error + return {"result": "success", "created_at": TimestampField().format(workflow.created_at)} @console_ns.route("/rag/pipelines//workflows/default-workflow-block-configs") @@ -739,7 +731,7 @@ def get(self, req_data: WorkflowListQuery, current_user: Account, pipeline: Pipe if user_id: if user_id != current_user.id: - raise Forbidden() + raise AccessDeniedError() rag_pipeline_service = RagPipelineService(db.session()) with sessionmaker(db.engine).begin() as session: @@ -752,44 +744,42 @@ def get(self, req_data: WorkflowListQuery, current_user: Account, pipeline: Pipe named_only=named_only, ) - return WorkflowPaginationResponse.model_validate( - { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], - "page": page, - "limit": limit, - "has_more": has_more, - } - ).model_dump(mode="json") + return WorkflowPaginationResponse.model_validate( + { + "items": workflows, + "page": page, + "limit": limit, + "has_more": has_more, + } + ).model_dump(mode="json") @console_ns.route("/rag/pipelines//workflows//restore") class RagPipelineDraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Success", console_ns.models[RagPipelineWorkflowSyncResponse.__name__]) - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def post(self, current_user: Account, pipeline: Pipeline, workflow_id: str): - rag_pipeline_service = RagPipelineService(db.session()) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def post(self, context: RequestContext, pipeline_id: str, workflow_id: str): try: - workflow = rag_pipeline_service.restore_published_workflow_to_draft( - pipeline=pipeline, - workflow_id=workflow_id, - account=current_user, + workflow = application_services().workflow_drafts.restore( + context, + WorkflowOwner(str(pipeline_id), "pipeline"), + workflow_id, ) + except WorkflowHashNotEqualError as exc: + raise DraftWorkflowNotSync() from exc except IsDraftWorkflowError as exc: # Use a stable, predefined message to keep the 400 response consistent - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -831,46 +821,24 @@ def patch(self, req_data: WorkflowUpdatePayload, current_user: Account, pipeline ) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") - return dump_response(WorkflowResponse, WorkflowResponseSource(workflow, session=session)) + return dump_response(WorkflowResponse, workflow) @console_ns.response(204, "Workflow deleted successfully") - @setup_required - @login_required - @account_initialization_required - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())) - @with_current_user - @get_rag_pipeline - def delete(self, current_user: Account, pipeline: Pipeline, workflow_id: str): - """ - Delete a published workflow version that is not currently active on the pipeline. - """ - if pipeline.workflow_id == workflow_id: - abort(400, description=f"Cannot delete workflow that is currently in use by pipeline '{pipeline.id}'") - - workflow_service = WorkflowService() - workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id) - - with sessionmaker(db.engine).begin() as session: - try: - retirement_candidates = workflow_service.delete_workflow( - session=session, - workflow_ref=workflow_ref, - ) - except WorkflowInUseError as e: - abort(400, description=str(e)) - except DraftWorkflowDeletionError as e: - abort(400, description=str(e)) - except ValueError as e: - raise NotFound(str(e)) from e - - WorkflowAgentRetirementService.retire_unowned( - tenant_id=pipeline.tenant_id, - agent_ids=retirement_candidates, - account_id=current_user.id, - ) + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.DATASET_EDIT, DatasetByPipeline())], + ) + def delete(self, context: RequestContext, pipeline_id: UUID, workflow_id: str): + try: + application_services().console_workflows.delete( + context, WorkflowOwner(str(pipeline_id), "pipeline"), workflow_id + ) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error return None, 204 @@ -1016,7 +984,7 @@ def get(self, session: Session, pipeline: Pipeline, run_id: UUID): rag_pipeline_service = RagPipelineService(session) workflow_run = rag_pipeline_service.get_rag_pipeline_workflow_run(pipeline=pipeline, run_id=run_id_str) if workflow_run is None: - raise NotFound("Workflow run not found") + raise NotFoundError("Workflow run not found") return WorkflowRunDetailResponse.model_validate( workflow_run_response_source(workflow_run, session=session), from_attributes=True @@ -1044,9 +1012,7 @@ def get(self, current_user: Account, pipeline: Pipeline, run_id: UUID): rag_pipeline_service = RagPipelineService(db.session()) user = cast("Account | EndUser", current_user) node_executions = rag_pipeline_service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, - run_id=run_id_str, - user=user, + pipeline=pipeline, run_id=run_id_str, user=user, tool_providers=application_services().tools.tool_providers ) return WorkflowRunNodeExecutionListResponse.model_validate( @@ -1083,16 +1049,19 @@ def get(self, pipeline: Pipeline, node_id: str): rag_pipeline_service = RagPipelineService(db.session()) workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") node_exec = rag_pipeline_service.get_node_last_run( pipeline=pipeline, workflow=workflow, node_id=node_id, ) if node_exec is None: - raise NotFound("last run not found") + raise NotFoundError("last run not found") return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=db.session()), from_attributes=True + node_execution_response_source( + node_exec, session=db.session(), tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -1110,15 +1079,15 @@ class RagPipelineTransformApi(Resource): def post(self, session: Session, current_tenant_id: str, current_user: Account, dataset_id: UUID): dataset = DatasetService.get_dataset_for_tenant(str(dataset_id), current_tenant_id, session=session) if dataset is None: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") if not dify_config.RBAC_ENABLED: if not (current_user.has_edit_permission or current_user.is_dataset_operator): - raise Forbidden() + raise AccessDeniedError() try: DatasetService.check_dataset_permission(dataset, current_user, session) except services.errors.base.NoPermissionError as exc: - raise Forbidden(str(exc)) from exc + raise AccessDeniedError(str(exc)) from exc try: return dump_response( @@ -1126,7 +1095,7 @@ def post(self, session: Session, current_tenant_id: str, current_user: Account, RagPipelineTransformService().transform_dataset(dataset, current_user.id, session), ) except RagPipelineResourceNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc @console_ns.route("/rag/pipelines//workflows/draft/datasource/variables-inspect") @@ -1151,14 +1120,18 @@ def post(self, req_data: DatasourceVariablesPayload, current_user: Account, pipe """ args = req_data.model_dump() - rag_pipeline_service = RagPipelineService(db.session()) - workflow_node_execution = rag_pipeline_service.set_datasource_variables( + workflow_node_execution = application_services().knowledge.pipeline_execution.set_datasource_variables( pipeline=pipeline, args=args, current_user=current_user, ) return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=db.session()), from_attributes=True + node_execution_response_source( + workflow_node_execution, + session=db.session(), + tool_providers=application_services().tools.tool_providers, + ), + from_attributes=True, ).model_dump(mode="json") @@ -1175,6 +1148,6 @@ class RagPipelineRecommendedPluginApi(Resource): def get(self, req_data: RagPipelineRecommendedPluginQuery, current_tenant_id: str, current_user: Account): rag_pipeline_service = RagPipelineService(db.session()) recommended_plugins = rag_pipeline_service.get_recommended_plugins( - req_data.type, current_user, current_tenant_id + req_data.type, current_user, current_tenant_id, tool_providers=application_services().tools.tool_providers ) return dump_response(RagPipelineRecommendedPluginsResponse, recommended_plugins, exclude_unset=True) diff --git a/api/controllers/console/explore/message.py b/api/controllers/console/explore/message.py index 2267f6c3ad7a54..fb3cb519524258 100644 --- a/api/controllers/console/explore/message.py +++ b/api/controllers/console/explore/message.py @@ -12,7 +12,11 @@ from pydantic import BaseModel from controllers.common.controller_schemas import MessageFeedbackPayload, MessageListQuery -from controllers.common.errors import InternalServerError, MessageFeedbackRatingRequiredError, UnauthorizedError +from controllers.common.errors import ( + AuthenticationRequiredError, + InternalServerError, + MessageFeedbackRatingRequiredError, +) from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console import console_ns from controllers.console.app.error import ( @@ -92,7 +96,7 @@ def decorated(*args: P.args, **kwargs: P.kwargs) -> R: except (AppDefinitionUnavailableError, MoreLikeThisConfigNotFoundError, AppModelConfigBrokenError) as error: raise AppUnavailableError() from error except (AccountNotFoundError, MessageActorNotFoundError) as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except MessageNotChatAppError as error: raise NotChatAppError() from error except MoreLikeThisNotCompletionError as error: diff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.py index 91e22d934d90ec..e76824ef1c87bb 100644 --- a/api/controllers/console/explore/trial.py +++ b/api/controllers/console/explore/trial.py @@ -15,6 +15,7 @@ from configs import dify_config from controllers.common.audio_response import audio_binary_response from controllers.common.errors import ( + AuthenticationRequiredError, BlockedFileExtensionError, FilenameNotExistsError, FileTooLargeError, @@ -28,7 +29,6 @@ RemoteFileUnavailableError, RemoteFileUrlBlockedError, TooManyFilesError, - UnauthorizedError, UnsupportedFileTypeError, ) from controllers.common.fields import ( @@ -81,7 +81,6 @@ from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file from controllers.console.wraps import cloud_edition_billing_resource_check, model_validate from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppGeneratorError from core.app.entities.app_invoke_entities import InvokeFrom from core.base.tts.audio_mime import SUPPORTED_TTS_AUDIO_MIME_TYPES from core.errors.error import ( @@ -112,6 +111,7 @@ from models.model import AppMode from services.account_errors import AccountNotFoundError from services.agent.errors import AgentVersionNotFoundError +from services.app.generation.errors import AgentAppGeneratorError from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_preview_query_service import ( AppPreviewOwnerUnavailableError, @@ -584,7 +584,7 @@ def post(self, req_data: WorkflowRunRequest, request_context: RequestContext, tr except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.app_model_config.AppModelConfigBrokenError as error: raise AppUnavailableError() from error except ProviderTokenNotInitError as ex: @@ -669,7 +669,7 @@ def post(self, req_data: ChatRequest, request_context: RequestContext, trial_app except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.conversation.ConversationNotExistsError as caught_error: raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: @@ -761,7 +761,7 @@ def get(self, request_context: RequestContext, trial_app: TrialAppRef, message_i except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except MessageActorNotFoundError as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except MessageNotExistsError as caught_error: raise NotFoundError("Message not found") from caught_error except ConversationNotExistsError as caught_error: @@ -945,7 +945,7 @@ def post(self, req_data: CompletionRequest, request_context: RequestContext, tri except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except services.errors.conversation.ConversationNotExistsError as caught_error: raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: @@ -1031,7 +1031,7 @@ def get(self, app: AppPreviewRef, request_context: RequestContext) -> dict[str, except AppDefinitionUnavailableError as error: raise AppUnavailableError() from error except AccountNotFoundError as error: - raise UnauthorizedError("Account no longer exists.") from error + raise AuthenticationRequiredError("Account no longer exists.") from error except AppPreviewSiteUnavailableError as error: raise AppPreviewSiteUnavailableHttpError(str(error)) from error diff --git a/api/controllers/console/snippets/snippet_workflow.py b/api/controllers/console/snippets/snippet_workflow.py index b6132f737e3cf3..0b9daad02ff649 100644 --- a/api/controllers/console/snippets/snippet_workflow.py +++ b/api/controllers/console/snippets/snippet_workflow.py @@ -6,25 +6,16 @@ from flask_restx import Resource from pydantic import BaseModel, Field from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound from controllers.common.controller_schemas import WorkflowUpdatePayload -from controllers.common.errors import InvalidArgumentError +from controllers.common.errors import InternalServerError, InvalidArgumentError, InvalidRequestError, NotFoundError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.rbac import RBACCheck, Workspace from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.common.session import with_session from controllers.console import console_ns from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync -from controllers.console.app.workflow import ( - RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, - DefaultBlockConfigsResponse, - WorkflowPaginationResponse, - WorkflowPublishResponse, - WorkflowResponse, - WorkflowResponseSource, - WorkflowRestoreResponse, -) +from controllers.console.flask_admission import console_account_admission from controllers.console.snippets.payloads import ( SnippetDraftNodeRunPayload, SnippetDraftRunPayload, @@ -42,10 +33,21 @@ model_validate, rbac_permission_required, setup_required, + validate_request, with_current_user, ) from core.app.entities.app_invoke_entities import InvokeFrom from core.db.session_factory import session_factory +from enums.account import TenantAccountRole +from extensions.application_services.snippets import build_snippet_service +from extensions.ext_application_services import application_services +from fields.workflow_fields import ( + DefaultBlockConfigsResponse, + WorkflowPaginationResponse, + WorkflowPublishResponse, + WorkflowResponse, + WorkflowRestoreResponse, +) from fields.workflow_run_fields import ( WorkflowRunDetailResponse, WorkflowRunNodeExecutionListResponse, @@ -59,15 +61,20 @@ from libs.external_api import error_responses from libs.helper import TimestampField from libs.login import current_account_with_tenant, login_required +from machinery.context import RequestContext from models import Account from models.model import AppMode from models.snippet import CustomizedSnippet -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.app_task_service import AppTaskService from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.snippet_generate_service import SnippetGenerateService from services.snippet_service import SnippetService +from services.workflow.contracts import ( + RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE, + DraftSyncCommand, + WorkflowOwner, +) logger = logging.getLogger(__name__) @@ -75,7 +82,7 @@ def _snippet_service() -> SnippetService: - return SnippetService(session_factory.get_session_maker()) + return build_snippet_service(session_factory.get_session_maker()) class SnippetWorkflowResponse(WorkflowResponse): @@ -149,7 +156,7 @@ def decorated_view(*args: P.args, **kwargs: P.kwargs) -> R: ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") kwargs["snippet"] = snippet @@ -177,19 +184,12 @@ class SnippetDraftWorkflowApi(Resource): def get(self, session: Session, snippet: CustomizedSnippet): """Get draft workflow for snippet.""" snippet_service = _snippet_service() - workflow = snippet_service.get_draft_workflow(snippet=snippet) + workflow = snippet_service.get_draft_workflow_record(snippet=snippet) if not workflow: raise DraftWorkflowNotExist() - workflow.conversation_variables = [] - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") - response["graph"] = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, - ) + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -202,26 +202,31 @@ def get(self, session: Session, snippet: CustomizedSnippet): console_ns.models[WorkflowRestoreResponse.__name__], ) @console_ns.response(400, "Hash mismatch") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - @model_validate(SnippetDraftSyncPayload) - def post(self, req_data: SnippetDraftSyncPayload, current_user: Account, snippet: CustomizedSnippet): + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def post(self, context: RequestContext, snippet_id: str): """Sync draft workflow for snippet.""" - + req_data = validate_request(SnippetDraftSyncPayload) try: - snippet_service = _snippet_service() - workflow = snippet_service.sync_draft_workflow( - snippet=snippet, - graph=req_data.graph, - unique_hash=req_data.hash, - account=current_user, - input_fields=req_data.input_fields, + workflow = application_services().workflow_drafts.sync( + context, + WorkflowOwner(str(snippet_id), "snippet"), + DraftSyncCommand( + graph=req_data.graph, + features={}, + unique_hash=req_data.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=req_data.input_fields, + ), ) + except WorkflowNotFoundError as exc: + raise NotFoundError(str(exc)) from exc except WorkflowHashNotEqualError as caught_error: raise DraftWorkflowNotSync() from caught_error except ValueError as e: @@ -229,7 +234,7 @@ def post(self, req_data: SnippetDraftSyncPayload, current_user: Account, snippet return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -275,14 +280,12 @@ def get(self, session: Session, snippet: CustomizedSnippet): return None snippet_service = _snippet_service() - workflow = snippet_service.get_published_workflow(snippet=snippet) + workflow = snippet_service.get_published_workflow_record(snippet=snippet) if not workflow: return None - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -294,26 +297,22 @@ def get(self, session: Session, snippet: CustomizedSnippet): @login_required @account_initialization_required @with_current_user - @with_session @get_snippet @edit_permission_required @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def post(self, session: Session, current_user: Account, snippet: CustomizedSnippet): + def post(self, current_user: Account, snippet: CustomizedSnippet): """Publish snippet workflow.""" snippet_service = _snippet_service() - snippet = session.merge(snippet) try: workflow = snippet_service.publish_workflow( - session=session, snippet=snippet, account=current_user, ) workflow_created_at = TimestampField().format(workflow.created_at) + except WorkflowHashNotEqualError as e: + raise DraftWorkflowNotSync() from e except ValueError as e: - # This domain validation remains a 400 response, so it cannot escape to - # the request-session decorator that normally owns rollback-on-error. - session.rollback() # guard-ignore: no-new-controller-sqlalchemy -- translated validation response raise InvalidArgumentError(str(e)) from e return { @@ -378,7 +377,7 @@ def get( response = SnippetWorkflowPaginationResponse.model_validate( { - "items": [WorkflowResponseSource(workflow, session=session) for workflow in workflows], + "items": workflows, "page": req_data.page, "limit": req_data.limit, "has_more": has_more, @@ -398,33 +397,31 @@ class SnippetDraftWorkflowRestoreApi(Resource): @console_ns.response(200, "Workflow restored successfully", console_ns.models[WorkflowRestoreResponse.__name__]) @console_ns.response(400, "Source workflow must be published") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @with_current_user - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def post(self, current_user: Account, snippet: CustomizedSnippet, workflow_id: str): + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def post(self, context: RequestContext, snippet_id: str, workflow_id: str): """Restore a published snippet workflow version into the draft workflow.""" - snippet_service = _snippet_service() try: - workflow = snippet_service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=workflow_id, - account=current_user, + workflow = application_services().workflow_drafts.restore( + context, + WorkflowOwner(str(snippet_id), "snippet"), + workflow_id, ) + except WorkflowHashNotEqualError as exc: + raise DraftWorkflowNotSync() from exc except IsDraftWorkflowError as exc: - raise BadRequest(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc + raise InvalidRequestError(RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE) from exc except WorkflowNotFoundError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc except ValueError as exc: - raise BadRequest(str(exc)) from exc + raise InvalidRequestError(str(exc)) from exc return { "result": "success", - "hash": workflow.unique_hash, + "hash": workflow.hash, "updated_at": TimestampField().format(workflow.updated_at or workflow.created_at), } @@ -470,11 +467,9 @@ def patch( data=update_data, ) if not workflow: - raise NotFound("Workflow not found") + raise NotFoundError("Workflow not found") - response = SnippetWorkflowResponse.model_validate( - WorkflowResponseSource(workflow, session=session), from_attributes=True - ).model_dump(mode="json") + response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json") response["input_fields"] = snippet.input_fields_list return response @@ -484,29 +479,20 @@ def patch( @console_ns.response(204, "Workflow deleted successfully") @console_ns.response(400, "Workflow is in use") @console_ns.response(404, "Workflow not found") - @setup_required - @login_required - @account_initialization_required - @get_snippet - @edit_permission_required - @rbac_permission_required(RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())) - def delete(self, snippet: CustomizedSnippet, workflow_id: str): - """Delete a published snippet workflow version.""" - snippet_service = _snippet_service() - with session_factory.get_session_maker().begin() as session: - try: - snippet_service.delete_workflow( - session=session, - snippet=snippet, - workflow_id=workflow_id, - ) - except WorkflowInUseError as e: - raise BadRequest(str(e)) from e - except DraftWorkflowDeletionError as e: - raise BadRequest(str(e)) from e - except ValueError as e: - raise NotFound(str(e)) from e - + @console_account_admission( + allowed_roles=frozenset({TenantAccountRole.OWNER, TenantAccountRole.ADMIN, TenantAccountRole.EDITOR}), + rbac_checks=[RBACCheck(RBACPermission.SNIPPETS_CREATE_AND_MODIFY, Workspace())], + ) + def delete(self, context: RequestContext, snippet_id: str, workflow_id: str): + """Delete a version and retire its unowned Agents through the shared use case.""" + try: + application_services().console_workflows.delete( + context, WorkflowOwner(str(snippet_id), "snippet"), str(workflow_id) + ) + except (WorkflowInUseError, DraftWorkflowDeletionError) as error: + raise InvalidRequestError(str(error)) from error + except WorkflowNotFoundError as error: + raise NotFoundError(str(error)) from error return None, 204 @@ -567,7 +553,7 @@ def get(self, session: Session, snippet: CustomizedSnippet, run_id): workflow_run = snippet_service.get_snippet_workflow_run(snippet=snippet, run_id=run_id) if not workflow_run: - raise NotFound("Workflow run not found") + raise NotFoundError("Workflow run not found") return WorkflowRunDetailResponse.model_validate( workflow_run_response_source(workflow_run, session=session), from_attributes=True @@ -592,8 +578,7 @@ def get(self, snippet: CustomizedSnippet, run_id): snippet_service = _snippet_service() node_executions = snippet_service.get_snippet_workflow_run_node_executions( - snippet=snippet, - run_id=run_id, + snippet=snippet, run_id=run_id, tool_providers=application_services().tools.tool_providers ) return WorkflowRunNodeExecutionListResponse.model_validate( @@ -638,23 +623,25 @@ def post( snippet_service = _snippet_service() draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) if not draft_workflow: - raise NotFound("Draft workflow not found") + raise NotFoundError("Draft workflow not found") files = SnippetGenerateService.parse_files(draft_workflow, req_data.files) - workflow_node_execution = SnippetGenerateService.run_draft_node( + workflow_node_execution = application_services().snippet_generation.run_draft_node( snippet=snippet, node_id=node_id, user_inputs=user_inputs, account=current_user, query=req_data.query, files=files, - session_maker=session_factory.get_session_maker(), ) with session_factory.create_session() as session: return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(workflow_node_execution, session=session), from_attributes=True + node_execution_response_source( + workflow_node_execution, session=session, tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -681,7 +668,7 @@ def get(self, snippet: CustomizedSnippet, node_id: str): snippet_service = _snippet_service() draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) if not draft_workflow: - raise NotFound("Draft workflow not found") + raise NotFoundError("Draft workflow not found") node_exec = snippet_service.get_snippet_node_last_run( snippet=snippet, @@ -689,11 +676,14 @@ def get(self, snippet: CustomizedSnippet, node_id: str): node_id=node_id, ) if node_exec is None: - raise NotFound("Node last run not found") + raise NotFoundError("Node last run not found") with session_factory.create_session() as session: return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(node_exec, session=session), from_attributes=True + node_execution_response_source( + node_exec, session=session, tool_providers=application_services().tools.tool_providers + ), + from_attributes=True, ).model_dump(mode="json") @@ -732,13 +722,12 @@ def post( args = req_data.model_dump(exclude_none=True) try: - response = SnippetGenerateService.generate_single_iteration( + response = application_services().snippet_generation.generate_single_iteration( snippet=snippet, user=current_user, node_id=node_id, args=args, streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) @@ -783,13 +772,12 @@ def post( """ try: - response = SnippetGenerateService.generate_single_loop( + response = application_services().snippet_generation.generate_single_loop( snippet=snippet, user=current_user, node_id=node_id, - args=req_data, + args=req_data.model_dump(), streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) @@ -827,13 +815,12 @@ def post(self, req_data: SnippetDraftRunPayload, current_user: Account, snippet: args = req_data.model_dump(exclude_none=True) try: - response = SnippetGenerateService.generate( + response = application_services().snippet_generation.generate( snippet=snippet, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, - session_maker=session_factory.get_session_maker(), ) return helper.compact_generate_response(response) diff --git a/api/controllers/console/workspace/snippets.py b/api/controllers/console/workspace/snippets.py index 4a2e693de1298d..96d5d9d651bec4 100644 --- a/api/controllers/console/workspace/snippets.py +++ b/api/controllers/console/workspace/snippets.py @@ -6,9 +6,8 @@ from flask import Response, request from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound -from controllers.common.errors import InvalidArgumentError +from controllers.common.errors import InvalidArgumentError, InvalidRequestError, NotFoundError from controllers.common.fields import TextFileResponse from controllers.common.rbac import RBACCheck, Workspace from controllers.common.schema import ( @@ -37,6 +36,7 @@ ) from core.db.session_factory import session_factory from core.plugin.entities.plugin import PluginDependency +from extensions.application_services.snippets import build_snippet_dsl_service, build_snippet_service from extensions.ext_application_services import application_services from fields.base import ResponseModel from fields.snippet_fields import ( @@ -51,7 +51,7 @@ from models import Account from models.snippet import SnippetType from services.entities.dsl_entities import DslImportWarning -from services.snippet_dsl_service import ImportStatus, SnippetDslService +from services.snippet_dsl_service import ImportStatus from services.snippet_service import SnippetService logger = logging.getLogger(__name__) @@ -78,8 +78,8 @@ class SnippetUseCountResponse(ResponseModel): def _snippet_service(session: Session | None = None) -> SnippetService: if session is not None: - return SnippetService(session=session) - return SnippetService(session_factory.get_session_maker()) + return build_snippet_service(session=session) + return build_snippet_service(session_factory.get_session_maker()) def _snippet_list_query_from_request() -> SnippetListQuery: @@ -181,9 +181,6 @@ def post( snippet_type = SnippetType.NODE try: - if req_data.graph is not None: - SnippetService.validate_snippet_graph_forbidden_nodes(req_data.graph) - snippet_service = _snippet_service(session) snippet = snippet_service.create_snippet( tenant_id=current_tenant_id, @@ -192,6 +189,7 @@ def post( snippet_type=snippet_type, icon_info=req_data.icon_info.model_dump() if req_data.icon_info else None, input_fields=[f.model_dump() for f in req_data.input_fields] if req_data.input_fields else None, + graph=req_data.graph, account=current_user, ) except ValueError as e: @@ -219,7 +217,7 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: UUID): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") return dump_response(SnippetResponse, snippet_response(snippet, session=session)), 200 @@ -253,7 +251,7 @@ def patch( ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") update_data = req_data.model_dump(exclude_unset=True) @@ -276,8 +274,8 @@ def patch( # Raise rather than return: `with_session` commits on a normal return, so returning here # would persist whatever the update wrote before it rejected the payload. Raising routes # through the decorator's rollback. Status stays 400 and `message` is unchanged; the body - # picks up the standard error envelope, as on every other BadRequest in the console API. - raise BadRequest(str(e)) from e + # picks up the standard error envelope, as on every other InvalidRequestError in the console API. + raise InvalidRequestError(str(e)) from e return dump_response(SnippetResponse, snippet_response(snippet, session=session)), 200 @@ -301,7 +299,7 @@ def delete(self, session: Session, current_tenant_id: str, current_user: Account ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") SnippetService.delete_snippet( session=session, @@ -336,12 +334,12 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") # Get include_secret parameter query = SnippetExportQuery.model_validate(request.args.to_dict()) - export_service = SnippetDslService(session) + export_service = build_snippet_dsl_service(session) try: result = export_service.export_snippet_dsl( snippet=snippet, @@ -349,7 +347,7 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): workflow_id=query.workflow_id, ) except ValueError as exc: - raise NotFound(str(exc)) from exc + raise NotFoundError(str(exc)) from exc # Set filename with .snippet extension filename = f"{snippet.name}.snippet" @@ -383,7 +381,7 @@ class CustomizedSnippetImportApi(Resource): @model_validate(SnippetImportPayload) def post(self, req_data: SnippetImportPayload, session: Session, current_user: Account): """Import snippet from DSL.""" - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.import_snippet( account=current_user, import_mode=req_data.mode, @@ -419,7 +417,7 @@ class CustomizedSnippetImportConfirmApi(Resource): @with_session def post(self, session: Session, current_user: Account, import_id: str): """Confirm a pending snippet import.""" - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.confirm_import(import_id=import_id, account=current_user) if result.status == ImportStatus.FAILED: @@ -454,9 +452,9 @@ def get(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") - import_service = SnippetDslService(session) + import_service = build_snippet_dsl_service(session) result = import_service.check_dependencies(snippet=snippet) return result.model_dump(mode="json"), 200 @@ -484,7 +482,7 @@ def post(self, session: Session, current_tenant_id: str, snippet_id: str): ) if not snippet: - raise NotFound("Snippet not found") + raise NotFoundError("Snippet not found") SnippetService.increment_use_count(session=session, snippet=snippet) session.flush() diff --git a/api/controllers/inner_api/agent/tools.py b/api/controllers/inner_api/agent/tools.py index be11c47486c0c7..19e42cd12504d4 100644 --- a/api/controllers/inner_api/agent/tools.py +++ b/api/controllers/inner_api/agent/tools.py @@ -2,14 +2,12 @@ from flask_restx import Resource from pydantic import ValidationError -from sqlalchemy.orm import Session from controllers.common.schema import register_response_schema_models, register_schema_models -from controllers.console.app.wraps import with_session from controllers.inner_api import inner_api_ns from controllers.inner_api.wraps import agent_inner_api_only +from extensions.ext_application_services import application_services from libs.exception import BaseHTTPException -from services.agent_tool_inner_service import AgentToolInnerService from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse from services.errors.agent_tool_inner import AgentToolInnerServiceError @@ -47,8 +45,7 @@ class AgentToolInvokeApi(Resource): @inner_api_ns.doc("inner_agent_tool_invoke") @inner_api_ns.expect(inner_api_ns.models[AgentToolInvokeRequest.__name__]) @inner_api_ns.response(200, "Tool invoked successfully", inner_api_ns.models[AgentToolInvokeResponse.__name__]) - @with_session - def post(self, session: Session) -> dict[str, object]: + def post(self) -> dict[str, object]: try: payload = AgentToolInvokeRequest.model_validate(inner_api_ns.payload or {}) except ValidationError as exc: @@ -59,7 +56,7 @@ def post(self, session: Session) -> dict[str, object]: ) from exc try: - response = AgentToolInnerService().invoke(session=session, request=payload) + response = application_services().agent_apps.tools.invoke(request=payload) except AgentToolInnerServiceError as exc: raise AgentToolInvokeHttpError( error_code=exc.error_code, diff --git a/api/controllers/inner_api/plugin/plugin.py b/api/controllers/inner_api/plugin/plugin.py index bb61e98e577076..008344f051f87a 100644 --- a/api/controllers/inner_api/plugin/plugin.py +++ b/api/controllers/inner_api/plugin/plugin.py @@ -9,12 +9,10 @@ from controllers.inner_api import inner_api_ns from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data from controllers.inner_api.wraps import plugin_inner_api_only -from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse from core.plugin.backwards_invocation.encrypt import PluginEncrypter from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation -from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation from core.plugin.entities.request import ( RequestFetchAppInfo, RequestInvokeApp, @@ -50,8 +48,9 @@ from models import Account, Tenant from models.model import EndUser from services.file_request_service import FileRequestService +from services.plugin.app_invocation import PluginAppBackwardsInvocation from services.plugin.datasource_backwards_invocation import PluginDatasourceBackwardsInvocation -from services.workflow_run_agg import WorkflowRunAgg +from services.tools.plugin_invocation import PluginToolBackwardsInvocation def _model_invocation_error_response(error: Exception) -> dict[str, object]: @@ -388,13 +387,10 @@ class PluginInvokeToolApi(Resource): 404: "Service not available", } ) - @with_session - def post(self, session: Session, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTool): + def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeTool): def generator(): return PluginToolBackwardsInvocation.convert_to_event_stream( PluginToolBackwardsInvocation.invoke_tool( - execution_driver=WorkflowRunAgg.run, - session=session, tenant_id=tenant_model.id, user_id=user_model.id, tool_type=ToolProviderType.value_of(payload.tool_type), @@ -402,6 +398,8 @@ def generator(): tool_name=payload.tool, tool_parameters=payload.tool_parameters, credential_id=payload.credential_id, + workflow_runtime=application_services().workflow_runtime, + draft_variable_saver=application_services().workflow_variables.saver_factory, ), ) @@ -492,7 +490,7 @@ class PluginInvokeAppApi(Resource): @with_session def post(self, session: Session, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeApp): response = PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, + runtime=application_services().workflow_runtime, session=session, app_id=payload.app_id, user_id=user_model.id, diff --git a/api/controllers/inner_api/runtime_credentials.py b/api/controllers/inner_api/runtime_credentials.py index 96144cb99c36d9..3ca4b179c22562 100644 --- a/api/controllers/inner_api/runtime_credentials.py +++ b/api/controllers/inner_api/runtime_credentials.py @@ -26,11 +26,11 @@ from core.helper.provider_cache import ToolProviderCredentialsCache from core.helper.provider_encryption import create_provider_encrypter from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager -from core.tools.tool_manager import ToolManager from extensions.ext_database import db from fields.base import ResponseModel from models.provider import ProviderCredential from models.tools import BuiltinToolProvider +from services.tools.tool_manager import ToolManager logger = logging.getLogger(__name__) diff --git a/api/controllers/mcp/mcp.py b/api/controllers/mcp/mcp.py index 23d0e3a8a2a4ec..8e1cd8869c3bf9 100644 --- a/api/controllers/mcp/mcp.py +++ b/api/controllers/mcp/mcp.py @@ -1,3 +1,4 @@ +from functools import partial from typing import Any, Literal, get_args from flask import Response, request @@ -12,12 +13,14 @@ from core.db.session_factory import session_factory from core.mcp import types as mcp_types from core.mcp.server.streamable_http import handle_mcp_request, negotiate_protocol_version +from extensions.ext_application_services import application_services from fields.base import ResponseModel from graphon.variables.input_entities import VariableEntity, VariableEntityType from libs import helper from models.enums import EndUserType from models.model import App, AppMCPServer, AppMode, EndUser from services.app.mcp_server_service import AppMCPServerStatus +from services.app_generate_service import AppGenerateService _MCP_METHODS = frozenset( method @@ -342,5 +345,17 @@ def _handle_mcp_request( end_user = self._create_end_user(client_name, app.tenant_id, app.id, mcp_server.id, session) return handle_mcp_request( - session, app, mcp_request, user_input_form, mcp_server, end_user, request_id, protocol_version + session, + app, + mcp_request, + user_input_form, + mcp_server, + end_user, + request_id, + protocol_version, + generate=partial( + AppGenerateService.generate, + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, + ), ) diff --git a/api/controllers/openapi/app_release.py b/api/controllers/openapi/app_release.py index c04f0f15fcc81a..aecfc16735611d 100644 --- a/api/controllers/openapi/app_release.py +++ b/api/controllers/openapi/app_release.py @@ -19,29 +19,30 @@ from controllers.openapi.app_workflow import VERSION_READ_GUARDS, require_draft from controllers.openapi.auth.context import Context from extensions.ext_application_services import application_services +from factories import variable_factory from graphon.enums import WorkflowExecutionStatus from models import AppMode, WorkflowRunTriggeredFrom -from models.workflow import Workflow, WorkflowRun +from models.workflow import WorkflowRun +from services.workflow.contracts import WorkflowOwner, WorkflowRecord from services.workflow.graph_check import check_graph, credential_check from services.workflow.graph_diff import WorkflowSnapshot, diff_workflows, same_graph -from services.workflow_service import WorkflowService -def _snapshot(workflow: Workflow) -> WorkflowSnapshot: +def _snapshot(workflow: WorkflowRecord) -> WorkflowSnapshot: return WorkflowSnapshot( - graph=workflow.graph_dict, - features=workflow.features_dict, + graph=workflow.graph, + features=workflow.features, environment_variable_names=frozenset(v.name for v in workflow.environment_variables), ) -def passed_on_draft(run: WorkflowRun | None, draft: Workflow) -> bool: +def passed_on_draft(run: WorkflowRun | None, draft: WorkflowRecord) -> bool: """The run passed on this draft. A run keeps only its graph, so a draft saved after the run started (environment variables, conversation variables, features) counts as untested.""" return ( run is not None and run.status == WorkflowExecutionStatus.SUCCEEDED - and same_graph(run.graph_dict, draft.graph_dict) + and same_graph(run.graph_dict, draft.graph) and draft.updated_at <= run.created_at ) @@ -59,12 +60,19 @@ class ReleaseCheckApi(Resource): def get(self, ctx: Context, app_id: str): draft = require_draft(ctx) mode = AppMode(ctx.app.mode) - service = WorkflowService() - environment = {v.name: v for v in draft.environment_variables} + environment = { + variable.name: variable + for variable in ( + variable_factory.build_environment_variable_from_mapping(value) + for value in application_services().console_workflow_variables.environment( + ctx.request_context, WorkflowOwner(ctx.app.id) + ) + ) + } issues = check_graph( - draft.graph_dict, + draft.graph, mode=mode, - resources=credential_check(ctx.workspace.id, environment, ctx.session), + resources=credential_check(ctx.workspace.id, environment, application_services().credential_queries), ) runs = ( application_services() @@ -77,7 +85,7 @@ def get(self, ctx: Context, app_id: str): .data ) tested = passed_on_draft(runs[0] if runs else None, draft) - published = service.get_published_workflow(ctx.app, session=ctx.session) + published = application_services().console_workflows.published(ctx.request_context, ctx.app.id) diff = diff_workflows(_snapshot(published) if published else None, _snapshot(draft)) checks = [ ReleaseCheckRow( diff --git a/api/controllers/openapi/app_run.py b/api/controllers/openapi/app_run.py index dce07e94145351..c451d06c65bfa1 100644 --- a/api/controllers/openapi/app_run.py +++ b/api/controllers/openapi/app_run.py @@ -10,16 +10,15 @@ from flask_restx import Resource from sqlalchemy.orm import Session -from werkzeug.exceptions import ( - BadRequest, - HTTPException, - InternalServerError, - NotFound, - TooManyRequests, -) import services from constants.oauth_bearer import Scope +from controllers.common.errors import ( + InvalidRequestError, + NotFoundError, + TooManyRequestsError, + raise_unexpected_error, +) from controllers.common.fields import EventStreamResponse from controllers.common.rbac import PlainApp, RBACCheck, RBACPermission from controllers.openapi import openapi_ns @@ -99,13 +98,13 @@ def _translate_service_errors() -> Generator[None]: try: yield except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) from ex + raise NotFoundError(str(ex)) from ex except (IsDraftWorkflowError, WorkflowIdFormatError) as ex: - raise BadRequest(str(ex)) from ex + raise InvalidRequestError(str(ex)) from ex except TriggerWorkflowServiceModeUnavailableServiceError as caught_error: raise TriggerWorkflowServiceModeUnavailableError() from caught_error except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: @@ -121,19 +120,16 @@ def _translate_service_errors() -> Generator[None]: # App concurrency limit. Without this it falls through to the bare `except Exception` # below and surfaces as a 500. Render as the canonical 429 (code "too_many_requests"); # the source message is dropped since it carries internal detail (client_id / limits). - raise TooManyRequests() from caught_error + raise TooManyRequestsError() from caught_error except InvokeRateLimitError as ex: raise InvokeRateLimitHttpError(ex.description) from ex except InvokeError as e: raise CompletionRequestError(e.description) from e except ValueError as caught_error: logger.warning("run input refused by the service.", exc_info=True) - raise BadRequest(_INVALID_RUN_INPUT) from caught_error - except HTTPException: - raise - except Exception as caught_error: - logger.exception("internal server error.") - raise InternalServerError() from caught_error + raise InvalidRequestError(_INVALID_RUN_INPUT) from caught_error + except Exception as error: + raise_unexpected_error(error, message="App execution failed") _RUN_GUARDS: Final = ( @@ -150,6 +146,8 @@ def _translate_service_errors() -> Generator[None]: def _generate(app: App, caller: Any, args: dict[str, Any], session: Session, invoke_from: InvokeFrom): return AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app, user=caller, @@ -489,5 +487,5 @@ def post(self, ctx: Context, app_id: str, task_id: str): if not application_services().app_tasks.stop_message_task( task_id=task_id, caller_role=ctx.subject.caller_role, caller_id=ctx.caller.id ): - raise NotFound("Task not found") + raise NotFoundError("Task not found") return TaskStopResponse(result="success") diff --git a/api/controllers/openapi/app_workflow.py b/api/controllers/openapi/app_workflow.py index 35d595a34c81a7..0966e4a2d6adaa 100644 --- a/api/controllers/openapi/app_workflow.py +++ b/api/controllers/openapi/app_workflow.py @@ -3,9 +3,9 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from http import HTTPStatus -from typing import Any, Final +from typing import Any, Final, cast from flask_restx import Resource from werkzeug.exceptions import BadRequest @@ -56,23 +56,20 @@ WorkflowRunDetailResponse, WorkflowRunNodeExecutionListResponse, WorkflowRunNodeExecutionResponse, - node_execution_response_source, workflow_run_pagination_response_source, workflow_run_response_source, ) from graphon.variables import SecretVariable, VariableBase from graphon.variables.exc import VariableError -from libs.datetime_utils import naive_utc_now from libs.helper import to_timestamp -from models import App, AppMode -from models.workflow import Workflow, WorkflowDataError, WorkflowRun +from models import AppMode +from models.workflow import Workflow, WorkflowRun from services.errors.app import IsDraftWorkflowError, WorkflowNotFoundError +from services.workflow.contracts import DraftWorkflowMissingError, WorkflowRecord from services.workflow.dsl_import import stored_secret_ids from services.workflow.graph_check import CONTAINER_NODE_TYPES, GRAPH_MODES from services.workflow.graph_diff import draft_token from services.workflow_run_service import WorkflowRunListArgs -from services.workflow_service import WorkflowService -from services.workflow_variable_reference_validator import advisory_variable_reference_warning _RUN_READ_GUARDS: Final = account_app_guards( RBACPermission.APP_CREATE_AND_MANAGEMENT, scope=Scope.APPS_READ, roles=None, modes=GRAPH_MODES @@ -191,24 +188,21 @@ class AppPublishApi(Resource): ) def post(self, ctx: Context, app_id: str, *, body: PublishPayload): try: - workflow = WorkflowService().publish_workflow( - session=ctx.session, - app_model=ctx.app, - account=ctx.account, + publication, warning = application_services().console_workflows.publish( + ctx.request_context, + ctx.app.id, marked_name=body.marked_name, marked_comment=body.marked_comment, ) except ValueError as exc: raise BadRequest(str(exc)) from exc - app_in_session = ctx.session.get(App, ctx.app.id) - if app_in_session: - app_in_session.workflow_id = workflow.id - app_in_session.updated_by = ctx.account.id - app_in_session.updated_at = naive_utc_now() + workflow = application_services().console_workflows.published(ctx.request_context, ctx.app.id) + if workflow is None: + raise RuntimeError("Published workflow is missing after publication") return PublishResponse( version_id=workflow.id, - created_at=to_timestamp(workflow.created_at), - warning=advisory_variable_reference_warning(workflow.graph), + created_at=to_timestamp(publication.created_at), + warning=warning, ) @@ -227,21 +221,28 @@ class AppVersionListApi(Resource): returns=(HTTPStatus.OK, VersionListResponse, "Version list"), ) def get(self, ctx: Context, app_id: str, *, query: VersionListQuery): - workflows, has_more = WorkflowService().get_all_published_workflow( - session=ctx.session, - app_model=ctx.app, + workflows, has_more = application_services().console_workflows.versions( + ctx.request_context, + ctx.app.id, page=query.page, limit=query.limit, user_id=None, named_only=query.named_only, - include_draft=False, ) + workflows = [workflow for workflow in workflows if workflow.version != Workflow.VERSION_DRAFT] page = VersionListResponse( page=query.page, limit=query.limit, has_more=has_more, data=[ - VersionRow.model_validate(workflow).model_copy(update={"current": workflow.id == ctx.app.workflow_id}) + VersionRow( + id=workflow.id, + marked_name=workflow.marked_name, + marked_comment=workflow.marked_comment, + created_by=workflow.created_by["id"] if workflow.created_by is not None else None, + created_at=to_timestamp(workflow.created_at), + current=workflow.id == ctx.app.workflow_id, + ) for workflow in workflows ], ) @@ -270,16 +271,14 @@ class AppVersionRestoreApi(Resource): def post(self, ctx: Context, app_id: str, version_id: str): version_id = require_uuid(version_id, not_found=VersionNotFound) try: - draft = WorkflowService().restore_published_workflow_to_draft( - app_model=ctx.app, workflow_id=version_id, account=ctx.account, session=ctx.session - ) + application_services().console_workflows.restore(ctx.request_context, ctx.app.id, version_id) except IsDraftWorkflowError as exc: raise VersionNotRestorable() from exc except WorkflowNotFoundError as exc: raise VersionNotFound() from exc except ValueError as exc: raise BadRequest(str(exc)) from exc - return RestoreResponse(result="success", draft_hash=draft_token(draft)) + return RestoreResponse(result="success", draft_hash=_draft_token(require_draft(ctx))) def env_variable_rows(variables: Iterable[VariableBase]) -> list[EnvVariableRow]: @@ -297,22 +296,37 @@ def env_variable_rows(variables: Iterable[VariableBase]) -> list[EnvVariableRow] ] -def require_draft(ctx: Context) -> Workflow: - draft = WorkflowService().get_draft_workflow(app_model=ctx.app, session=ctx.session) - if draft is None: - raise DraftNotFound() - return draft +def require_draft(ctx: Context) -> WorkflowRecord: + try: + return application_services().console_workflows.draft(ctx.request_context, ctx.app.id) + except DraftWorkflowMissingError as exc: + raise DraftNotFound() from exc + + +class _DraftTokenView: + """Expose the immutable workflow projection in the shape the legacy token helper reads.""" + + def __init__(self, workflow: WorkflowRecord) -> None: + self.graph_dict = workflow.graph + self.features_dict = workflow.features + self.environment_variables = workflow.environment_variables + self.conversation_variables = workflow.conversation_variables + + +def _draft_token(draft: WorkflowRecord) -> str: + return draft_token(cast(Any, _DraftTokenView(draft))) def _patch_env(ctx: Context, *, upserts: list[VariableBase], deletions: list[str]) -> None: try: - WorkflowService().patch_draft_workflow_environment_variables( - app_model=ctx.app, - environment_variables=upserts, - deleted_environment_variable_ids=deletions, - account=ctx.account, - session=ctx.session, + application_services().console_workflow_variables.update_environment( + ctx.request_context, + ctx.app.id, + [variable.model_dump(mode="json") for variable in upserts], + deleted_ids=deletions, ) + except DraftWorkflowMissingError as exc: + raise DraftNotFound() from exc except ValueError as exc: raise BadRequest(str(exc)) from exc @@ -392,33 +406,35 @@ def _node_run_guards(mode: AppMode) -> tuple[Requirement, ...]: def run_draft_node( ctx: Context, node_id: str, *, inputs: dict[str, Any], query: str, files: UploadParts | None = None ) -> WorkflowRunNodeExecutionResponse: - workflow_service = WorkflowService() draft = require_draft(ctx) try: - node = draft.get_node_config_by_id(node_id) - except WorkflowDataError as exc: + nodes = draft.graph.get("nodes") + if not isinstance(nodes, list): + raise LookupError + node = next(node for node in nodes if isinstance(node, Mapping) and node.get("id") == node_id) + data = node.get("data") + if not isinstance(data, Mapping): + raise LookupError + node_type = data.get("type") + if not isinstance(node_type, str): + raise LookupError + except (LookupError, StopIteration) as exc: raise NodeNotFound() from exc - except ValueError as exc: - raise BadRequest(str(exc)) from exc - if Workflow.get_node_type_from_node_config(node) in CONTAINER_NODE_TYPES: + if node_type in CONTAINER_NODE_TYPES: raise BadRequest("Loop and iteration nodes can't run alone; test them with a full draft run") if files: end_read_transaction(ctx.session) user_inputs = merge_files(inputs, files, ctx.caller) try: - execution = workflow_service.run_draft_workflow_node( - app_model=ctx.app, - draft_workflow=draft, - node_id=node_id, - user_inputs=user_inputs, - account=ctx.account, - query=query, + execution = application_services().console_workflows.run_node( + ctx.request_context, + ctx.app.id, + node_id, + {"inputs": user_inputs, "query": query}, ) except ValueError as exc: raise BadRequest(str(exc)) from exc - return WorkflowRunNodeExecutionResponse.model_validate( - node_execution_response_source(execution, session=ctx.session), from_attributes=True - ) + return WorkflowRunNodeExecutionResponse.model_validate(execution) @openapi_ns.route("/apps//draft/workflow/nodes/:run") diff --git a/api/controllers/service_api/app/completion.py b/api/controllers/service_api/app/completion.py index b42c4efa1d4fb5..5c525193b399a6 100644 --- a/api/controllers/service_api/app/completion.py +++ b/api/controllers/service_api/app/completion.py @@ -7,10 +7,10 @@ from pydantic import BaseModel, Field, field_validator from pydantic.json_schema import SkipJsonSchema from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services from configs import dify_config +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import ChatBlockingResponse, CompletionBlockingResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -35,7 +35,6 @@ ) from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -44,11 +43,13 @@ ) from core.helper.trace_id_helper import get_external_trace_id, get_trace_session_id, omit_trace_session_id_from_payload from enums import CloudPlan, DeploymentEdition +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.external_api import error_responses from libs.helper import UUIDStrOrEmpty from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.billing_service import BillingService @@ -64,7 +65,7 @@ def _resolve_agent_app_streaming(*, app_mode: AppMode, response_mode: str | None if app_mode != AppMode.AGENT: return response_mode == "streaming" if response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -252,6 +253,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -263,7 +266,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: @@ -444,6 +447,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): ) response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -455,13 +460,13 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) from ex + raise NotFoundError(str(ex)) from ex except IsDraftWorkflowError as ex: - raise BadRequest(str(ex)) from ex + raise InvalidRequestError(str(ex)) from ex except WorkflowIdFormatError as ex: - raise BadRequest(str(ex)) from ex + raise InvalidRequestError(str(ex)) from ex except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: diff --git a/api/controllers/service_api/app/workflow.py b/api/controllers/service_api/app/workflow.py index 1e11a4aa67e79a..2bed188ede2310 100644 --- a/api/controllers/service_api/app/workflow.py +++ b/api/controllers/service_api/app/workflow.py @@ -9,10 +9,10 @@ from pydantic import BaseModel, Field, WithJsonSchema, field_validator, model_validator from pydantic.json_schema import SkipJsonSchema from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound from configs import dify_config from controllers.common.controller_schemas import WorkflowRunPayload as WorkflowRunPayloadBase +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import SimpleResultResponse, WorkflowBlockingResponse from controllers.common.schema import ( query_params_from_model, @@ -280,7 +280,7 @@ def get(self, app_model: App, workflow_run_id: str): run_id=workflow_run_id, ) if not workflow_run: - raise NotFound("Workflow run not found.") + raise NotFoundError("Workflow run not found.") return dump_response(WorkflowRunResponse, workflow_run) @@ -371,6 +371,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -513,6 +515,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser, workflow_id: try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -526,11 +530,11 @@ def post(self, session: Session, app_model: App, end_user: EndUser, workflow_id: except TriggerWorkflowServiceModeUnavailableServiceError as caught_error: raise TriggerWorkflowServiceModeUnavailableError() from caught_error except WorkflowNotFoundError as ex: - raise NotFound(str(ex)) from ex + raise NotFoundError(str(ex)) from ex except IsDraftWorkflowError as ex: - raise BadRequest(str(ex)) from ex + raise InvalidRequestError(str(ex)) from ex except WorkflowIdFormatError as ex: - raise BadRequest(str(ex)) from ex + raise InvalidRequestError(str(ex)) from ex except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) from ex except QuotaExceededError as caught_error: diff --git a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py index 287218d342ec99..2ec35d96cfb2fa 100644 --- a/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py +++ b/api/controllers/service_api/dataset/rag_pipeline/rag_pipeline_workflow.py @@ -7,13 +7,14 @@ from pydantic import BaseModel, Field, RootModel, field_validator from sqlalchemy import select from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound import services from controllers.common.errors import ( + AccessDeniedError, FilenameNotExistsError, FileTooLargeError, NoFileUploadedError, + NotFoundError, TooManyFilesError, UnsupportedFileTypeError, ) @@ -30,7 +31,6 @@ from controllers.service_api.dataset.error import PipelineRunError from controllers.service_api.schema import event_stream_response, json_or_event_stream_response, multipart_file_params from controllers.service_api.wraps import DatasetApiResource -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.entities.knowledge_entities import PipelineDataset, PipelineDocument from extensions.ext_application_services import application_services @@ -42,6 +42,7 @@ from models import Account from models.dataset import Dataset, Pipeline from models.engine import db +from services.app.generation.response import convert_to_event_stream from services.feature_service import FeatureService from services.file_service import FileService from services.rag_pipeline.entity.pipeline_service_api_entities import ( @@ -179,7 +180,7 @@ def get(self, tenant_id: str, dataset_id: UUID): stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = db.session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") query = query_params_from_request(DatasourcePluginsQuery) @@ -230,7 +231,7 @@ def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UU stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = db.session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") assert isinstance(current_user, Account) rag_pipeline_service: RagPipelineService = RagPipelineService(db.session()) @@ -244,8 +245,8 @@ def post(self, payload: DatasourceNodeRunPayload, tenant_id: str, dataset_id: UU ) # response-contract:ignore compact_generate_response return helper.compact_generate_response( - PipelineGenerator.convert_to_event_stream( - rag_pipeline_service.run_datasource_workflow_node( + convert_to_event_stream( + application_services().knowledge.pipeline_execution.run_datasource_workflow_node( pipeline=pipeline, node_id=node_id, user_inputs=datasource_node_run_api_entity.inputs, @@ -277,7 +278,7 @@ class PipelineRunApi(DatasetApiResource): "`documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON " "object for blocking mode." ), - 403: "`forbidden` : Forbidden.", + 403: "`forbidden` : AccessDeniedError.", 404: "`not_found` : Dataset not found.", 500: "`pipeline_run_error` : Pipeline execution failed.", }, @@ -307,19 +308,19 @@ def post(self, session: Session, tenant_id: str, dataset_id: UUID): stmt = select(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id_str) dataset = session.scalar(stmt) if not dataset: - raise NotFound("Dataset not found.") + raise NotFoundError("Dataset not found.") payload = PipelineRunApiEntity.model_validate(service_api_ns.payload or {}) if not isinstance(current_user, Account): - raise Forbidden() + raise AccessDeniedError() rag_pipeline_service = RagPipelineService(session) pipeline = rag_pipeline_service.get_pipeline(tenant_id=tenant_id, dataset_id=dataset_id_str) try: response: dict[Any, Any] | Generator[str, Any] = PipelineGenerateService.generate( - generator=application_services().knowledge.pipeline_generator, session=session, + generator=application_services().knowledge.pipeline_generator, pipeline=pipeline, user=current_user, args=payload.model_dump(), diff --git a/api/controllers/trigger/__init__.py b/api/controllers/trigger/__init__.py index 51829575b9ba80..16e1c4030ad5ab 100644 --- a/api/controllers/trigger/__init__.py +++ b/api/controllers/trigger/__init__.py @@ -2,8 +2,11 @@ from flask import Blueprint +from controllers.trigger.errors import register_error_handlers + # Create trigger blueprint bp = Blueprint("trigger", __name__, url_prefix="/triggers") +register_error_handlers(bp) def _register_controllers() -> tuple[ModuleType, ...]: diff --git a/api/controllers/trigger/errors.py b/api/controllers/trigger/errors.py new file mode 100644 index 00000000000000..9a88fd51215105 --- /dev/null +++ b/api/controllers/trigger/errors.py @@ -0,0 +1,25 @@ +"""Flask error responses for trigger endpoints. + +Request parsing failures retain their HTTP status and response. Route handlers +translate domain failures; unexpected failures are rendered here after unwinding +the handler, so they cannot swallow a framework-generated 400 or 413. +""" + +import logging + +from flask import Blueprint, Response, jsonify, request +from werkzeug.exceptions import HTTPException + +logger = logging.getLogger(__name__) + + +def register_error_handlers(blueprint: Blueprint) -> None: + @blueprint.errorhandler(HTTPException) + def handle_http_error(error: HTTPException) -> HTTPException: + return error + + @blueprint.errorhandler(Exception) + def handle_unexpected_error(error: Exception) -> tuple[Response, int]: + logger.exception("Trigger request failed for %s", request.path) + message = str(error) if request.endpoint == "trigger.handle_webhook" else "An internal error has occurred." + return jsonify({"error": "Internal server error", "message": message}), 500 diff --git a/api/controllers/trigger/webhook.py b/api/controllers/trigger/webhook.py index b9734cb9f39db7..c753e3553e60d5 100644 --- a/api/controllers/trigger/webhook.py +++ b/api/controllers/trigger/webhook.py @@ -5,12 +5,14 @@ from werkzeug.exceptions import RequestEntityTooLarge from controllers.common.access_response import webhook_not_found_response +from controllers.common.errors import RequestBodyTooLargeError from controllers.trigger import bp from core.trigger.debug.event_bus import TriggerDebugEventBus from core.trigger.debug.events import WebhookDebugEvent, build_webhook_pool_key from enums import QuotaType from extensions.ext_application_services import application_services from services.errors.app import QuotaExceededError +from services.trigger.errors import WebhookBodyTooLargeError from services.trigger.webhook_service import RawWebhookDataDict, WebhookService logger = logging.getLogger(__name__) @@ -92,11 +94,8 @@ def handle_webhook(webhook_id: str): ), 429 except ValueError: return webhook_not_found_response() - except RequestEntityTooLarge: - raise - except Exception: - logger.exception("Webhook processing failed for %s", webhook_id) - return jsonify({"error": "Internal server error", "message": "An internal error has occurred."}), 500 + except WebhookBodyTooLargeError as error: + raise RequestBodyTooLargeError(str(error)) from error @bp.route("/webhook-debug/", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"]) @@ -163,6 +162,8 @@ def handle_webhook_debug(webhook_id: str): except ValueError: return webhook_not_found_response() + except WebhookBodyTooLargeError as error: + raise RequestBodyTooLargeError(str(error)) from error except RequestEntityTooLarge: raise except Exception: diff --git a/api/controllers/web/completion.py b/api/controllers/web/completion.py index 4918b61ea033bd..b8ca78c95a8886 100644 --- a/api/controllers/web/completion.py +++ b/api/controllers/web/completion.py @@ -3,9 +3,9 @@ from pydantic import BaseModel, Field, field_validator from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, InternalServerError, NotFound import services +from controllers.common.errors import InternalServerError, InvalidRequestError, NotFoundError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -23,18 +23,19 @@ ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError from controllers.web.wraps import WebApiResource -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError, ) +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.external_api import error_responses from libs.helper import uuid_value from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.conversation_service import ConversationService @@ -48,7 +49,7 @@ def _resolve_agent_app_streaming(*, app_mode: AppMode, response_mode: str | None if app_mode != AppMode.AGENT: return response_mode == "streaming" if response_mode == "blocking": - raise BadRequest("Agent App only supports streaming response mode.") + raise InvalidRequestError("Agent App only supports streaming response mode.") return True @@ -132,6 +133,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -143,7 +146,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: @@ -246,6 +249,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): ) response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, @@ -257,7 +262,7 @@ def post(self, session: Session, app_model: App, end_user: EndUser): # response-contract:ignore compact_generate_response return helper.compact_generate_response(response) except services.errors.conversation.ConversationNotExistsError as caught_error: - raise NotFound("Conversation Not Exists.") from caught_error + raise NotFoundError("Conversation Not Exists.") from caught_error except services.errors.conversation.ConversationCompletedError as caught_error: raise ConversationCompletedError() from caught_error except services.errors.app_model_config.AppModelConfigBrokenError as caught_error: diff --git a/api/controllers/web/workflow.py b/api/controllers/web/workflow.py index 0672eeb155d914..0d4505541f6412 100644 --- a/api/controllers/web/workflow.py +++ b/api/controllers/web/workflow.py @@ -1,9 +1,9 @@ import logging from sqlalchemy.orm import Session -from werkzeug.exceptions import InternalServerError from controllers.common.controller_schemas import WorkflowRunPayload +from controllers.common.errors import InternalServerError from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse from controllers.common.schema import register_response_schema_models, register_schema_models from controllers.console.app.wraps import with_session @@ -24,6 +24,7 @@ ProviderTokenNotInitError, QuotaExceededError, ) +from extensions.ext_application_services import application_services from graphon.model_runtime.errors.invoke import InvokeError from libs import helper from libs.external_api import error_responses @@ -80,6 +81,8 @@ def post(self, session: Session, app_model: App, end_user: EndUser): try: response = AppGenerateService.generate( + variables=application_services().workflow_variables, + runtime=application_services().workflow_runtime, session=session, app_model=app_model, user=end_user, diff --git a/api/core/app/apps/agent_app/session_store.py b/api/core/app/apps/agent_app/session_store.py index b1c5110900f466..bb86491cfd7cbf 100644 --- a/api/core/app/apps/agent_app/session_store.py +++ b/api/core/app/apps/agent_app/session_store.py @@ -17,6 +17,7 @@ AgentWorkspaceOwnerType, ) from models.model import App, Conversation +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.workspace_service import ( AgentWorkspaceNotFoundError, AgentWorkspaceService, @@ -126,7 +127,7 @@ def _get_binding( ) if binding is None or binding.agent_id != scope.agent_id: raise AgentWorkspaceNotFoundError("Caller participant Binding is unavailable") - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=scope.home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, diff --git a/api/core/app/features/annotation_reply/annotation_reply.py b/api/core/app/features/annotation_reply/annotation_reply.py index 454a239494256c..4cf21bed4d6c81 100644 --- a/api/core/app/features/annotation_reply/annotation_reply.py +++ b/api/core/app/features/annotation_reply/annotation_reply.py @@ -9,8 +9,8 @@ from models.dataset import Dataset from models.enums import CollectionBindingType, ConversationFromSource from models.model import AnnotationReplyEnabledConfig, App, Message, MessageAnnotation, load_annotation_reply_config +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.annotation_service import AppAnnotationService -from services.knowledge.dataset_service import DatasetCollectionBindingService logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ def query( embedding_provider_name = enabled_config["embedding_model"]["embedding_provider_name"] embedding_model_name = enabled_config["embedding_model"]["embedding_model_name"] - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_provider_name, embedding_model_name, session, CollectionBindingType.ANNOTATION ) diff --git a/api/core/app/task_pipeline/based_generate_task_pipeline.py b/api/core/app/task_pipeline/based_generate_task_pipeline.py index 925e13f274c334..1ff89b92db82c7 100644 --- a/api/core/app/task_pipeline/based_generate_task_pipeline.py +++ b/api/core/app/task_pipeline/based_generate_task_pipeline.py @@ -47,7 +47,9 @@ def __init__( self.output_moderation_handler = self._init_output_moderation() self.stream = stream - def handle_error(self, *, event: QueueErrorEvent, session: Session | None = None, message_id: str = ""): + def handle_error( + self, *, event: QueueErrorEvent, session: Session | None = None, message_id: str = "" + ) -> Exception: logger.debug("error: %s", event.error) e = event.error err: Exception @@ -61,17 +63,15 @@ def handle_error(self, *, event: QueueErrorEvent, session: Session | None = None description = getattr(e, "description", None) err = Exception(description if description is not None else str(e)) - if not message_id or not session: + if not message_id or session is None: return err - stmt = select(Message).where(Message.id == message_id) - message = session.scalar(stmt) - if not message: + message = session.scalar(select(Message).where(Message.id == message_id)) + if message is None: return err - err_desc = self._error_to_desc(err) message.status = MessageStatus.ERROR - message.error = err_desc + message.error = self._error_to_desc(err) return err def _error_to_desc(self, e: Exception) -> str: diff --git a/api/core/app/workflow/__init__.py b/api/core/app/workflow/__init__.py index 3bca7f5c34abf6..e69de29bb2d1d6 100644 --- a/api/core/app/workflow/__init__.py +++ b/api/core/app/workflow/__init__.py @@ -1,3 +0,0 @@ -from core.workflow.node_factory import DifyNodeFactory - -__all__ = ["DifyNodeFactory"] diff --git a/api/core/app/workflow/file_runtime.py b/api/core/app/workflow/file_runtime.py index 886ef5e52d0180..cd33a762253a1e 100644 --- a/api/core/app/workflow/file_runtime.py +++ b/api/core/app/workflow/file_runtime.py @@ -3,7 +3,6 @@ import base64 import hashlib import hmac -import os import time import urllib.parse from collections.abc import Generator @@ -19,6 +18,7 @@ from graphon.file import FileTransferMethod from graphon.file.protocols import WorkflowFileRuntimeProtocol from graphon.http.protocols import HttpResponseProtocol +from libs.signed_query import sign_query if TYPE_CHECKING: from dify_app import DifyApp @@ -121,11 +121,10 @@ def resolve_upload_file_uri( """Resolve a signed UploadFile URI without selecting an origin.""" self._assert_upload_file_access(upload_file_id=upload_file_id) - uri = f"/files/{upload_file_id}/file-preview" - query = self._sign_query(payload=f"file-preview|{upload_file_id}") + query = sign_query(payload=f"file-preview|{upload_file_id}", key=self._secret_key()) if as_attachment: query["as_attachment"] = "true" - return f"{uri}?{urllib.parse.urlencode(query)}" + return f"/files/{upload_file_id}/file-preview?{urllib.parse.urlencode(query)}" @override def resolve_tool_file_url(self, *, tool_file_id: str, extension: str, for_external: bool = True) -> str: @@ -164,16 +163,6 @@ def _base_url(*, for_external: bool) -> str: def _secret_key() -> bytes: return dify_config.SECRET_KEY.encode() - def _sign_query(self, *, payload: str) -> dict[str, str]: - timestamp = str(int(time.time())) - nonce = os.urandom(16).hex() - sign = hmac.new(self._secret_key(), f"{payload}|{timestamp}|{nonce}".encode(), hashlib.sha256).digest() - return { - "timestamp": timestamp, - "nonce": nonce, - "sign": base64.urlsafe_b64encode(sign).decode(), - } - def _resolve_storage_key(self, *, file: File) -> str: parsed_reference = parse_file_reference(file.reference) if parsed_reference is None: diff --git a/api/core/app/workflow/layers/__init__.py b/api/core/app/workflow/layers/__init__.py index 8e69b33a9403d5..9c2c285b102ef7 100644 --- a/api/core/app/workflow/layers/__init__.py +++ b/api/core/app/workflow/layers/__init__.py @@ -1 +1,5 @@ """Workflow Engine observability layer.""" + +from .observability import ObservabilityLayer + +__all__ = ["ObservabilityLayer"] diff --git a/api/core/mcp/server/streamable_http.py b/api/core/mcp/server/streamable_http.py index 3b96c5da43eb3e..c0372ee45fdc8f 100644 --- a/api/core/mcp/server/streamable_http.py +++ b/api/core/mcp/server/streamable_http.py @@ -1,6 +1,6 @@ import json import logging -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any, NotRequired, TypedDict, cast from sqlalchemy.orm import Session @@ -11,7 +11,6 @@ from core.mcp import types as mcp_types from graphon.variables.input_entities import VariableEntity, VariableEntityType from models.model import App, AppMCPServer, AppMode, EndUser -from services.app_generate_service import AppGenerateService from services.errors.app import TriggerWorkflowServiceModeUnavailableError logger = logging.getLogger(__name__) @@ -67,6 +66,8 @@ def handle_mcp_request( end_user: EndUser | None = None, request_id: int | str = 1, protocol_version: str = mcp_types.DEFAULT_NEGOTIATED_VERSION, + *, + generate: Callable[..., Any], ) -> mcp_types.JSONRPCResponse | mcp_types.JSONRPCError: """ Handle MCP request and return JSON-RPC response @@ -130,7 +131,9 @@ def create_error_response( ) case mcp_types.CallToolRequest(): return create_success_response( - handle_call_tool(session, app, request, user_input_form, end_user, protocol_version) + handle_call_tool( + session, app, request, user_input_form, end_user, protocol_version, generate=generate + ) ) case mcp_types.PingRequest(): return create_success_response(handle_ping()) @@ -210,6 +213,8 @@ def handle_call_tool( user_input_form: list[VariableEntity], end_user: EndUser | None, protocol_version: str = mcp_types.DEFAULT_NEGOTIATED_VERSION, + *, + generate: Callable[..., Any], ) -> mcp_types.CallToolResult: """Handle call tool request""" request_obj = cast(mcp_types.CallToolRequest, request.root) @@ -218,7 +223,7 @@ def handle_call_tool( if not end_user: raise ValueError("End user not found") - response = AppGenerateService.generate( + response = generate( session=session, app_model=app, user=end_user, diff --git a/api/core/plugin/backwards_invocation/node.py b/api/core/plugin/backwards_invocation/node.py index 9550e499922cfb..86458869aba07f 100644 --- a/api/core/plugin/backwards_invocation/node.py +++ b/api/core/plugin/backwards_invocation/node.py @@ -34,7 +34,6 @@ def invoke_parameter_extractor( :param query: str :return: dict """ - # FIXME(-LAN-): Avoid import service into core workflow_service = WorkflowService() node_id = "1919810" node_data = ParameterExtractorNodeData( @@ -85,7 +84,6 @@ def invoke_question_classifier( :param query: str :return: dict """ - # FIXME(-LAN-): Avoid import service into core workflow_service = WorkflowService() node_id = "1919810" node_data = QuestionClassifierNodeData( diff --git a/api/core/tools/workflow_as_tool/repository.py b/api/core/tools/workflow_as_tool/repository.py index 05f082a6201f0b..2bd17178f57831 100644 --- a/api/core/tools/workflow_as_tool/repository.py +++ b/api/core/tools/workflow_as_tool/repository.py @@ -1,13 +1,11 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from typing import Any +from core.tools.__base.tool import Tool from enums import WorkflowKind from graphon.variables import VariableBase -if TYPE_CHECKING: - from core.tools.workflow_as_tool.tool import WorkflowTool - @dataclass(frozen=True, slots=True) class WorkflowToolSource: @@ -19,4 +17,4 @@ class WorkflowToolSource: features_dict: Mapping[str, Any] environment_variables: Sequence[VariableBase] workflow_kind: WorkflowKind - tool: "WorkflowTool | None" = None + tool: Tool | None = None diff --git a/api/core/workflow/node_factory.py b/api/core/workflow/node_factory.py index 1eca01664b516b..397fb4523492ba 100644 --- a/api/core/workflow/node_factory.py +++ b/api/core/workflow/node_factory.py @@ -48,7 +48,7 @@ PluginAgentStrategyResolver, ) from core.workflow.nodes.agent.runtime_support import AgentRuntimeSupport -from core.workflow.nodes.agent_v2 import DifyAgentNode +from core.workflow.nodes.agent_v2.agent_node import DifyAgentNode from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data from core.workflow.nodes.agent_v2.output_adapter import WorkflowAgentOutputAdapter @@ -421,7 +421,9 @@ def __init__( self._agent_strategy_resolver = PluginAgentStrategyResolver() self._agent_strategy_presentation_provider = PluginAgentStrategyPresentationProvider() self._agent_runtime_support = AgentRuntimeSupport() - self._agent_message_transformer = AgentMessageTransformer() + self._agent_message_transformer = AgentMessageTransformer( + lambda name, default: self._tool_runtime.resolve_provider_icons(provider_name=name, default_icon=default) + ) @override def with_runtime_state(self, runtime_state: "RuntimeState") -> "DifyNodeFactory": diff --git a/api/core/workflow/nodes/agent/__init__.py b/api/core/workflow/nodes/agent/__init__.py index ba6c667194ab16..7967b3395ae830 100644 --- a/api/core/workflow/nodes/agent/__init__.py +++ b/api/core/workflow/nodes/agent/__init__.py @@ -1,4 +1,3 @@ -from .agent_node import AgentNode from .entities import AgentNodeData -__all__ = ["AgentNode", "AgentNodeData"] +__all__ = ["AgentNodeData"] diff --git a/api/core/workflow/nodes/agent/message_transformer.py b/api/core/workflow/nodes/agent/message_transformer.py index aad3ee805c2012..816b3973cccc5f 100644 --- a/api/core/workflow/nodes/agent/message_transformer.py +++ b/api/core/workflow/nodes/agent/message_transformer.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Generator, Mapping +from collections.abc import Callable, Generator, Mapping from typing import Any, cast from sqlalchemy import select @@ -23,7 +23,6 @@ ) from graphon.variables.segments import ArrayFileSegment from models import ToolFile -from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService from .events import AgentLogEvent from .exceptions import AgentNodeError, AgentVariableTypeError, ToolFileNotFoundError @@ -33,6 +32,14 @@ class AgentMessageTransformer: + def __init__( + self, + provider_icons: Callable[ + [str, str | None], tuple[str | Mapping[str, str] | None, str | Mapping[str, str] | None] + ], + ) -> None: + self._provider_icons = provider_icons + def transform( self, *, @@ -46,8 +53,6 @@ def transform( node_id: str, node_execution_id: str, ) -> Generator[NodeEventPayload]: - from core.plugin.impl.plugin import PluginInstaller - message_stream = ToolFileMessageTransformer.transform_tool_invoke_messages( messages=messages, user_id=user_id, @@ -199,32 +204,7 @@ def transform( icon = tool_info.get("icon", "") dict_metadata = dict(message.message.metadata) if dict_metadata.get("provider"): - manager = PluginInstaller() - plugins = manager.list_plugins(tenant_id) - try: - current_plugin = next( - plugin - for plugin in plugins - if f"{plugin.plugin_id}/{plugin.name}" == dict_metadata["provider"] - ) - icon = current_plugin.declaration.icon - except StopIteration: - pass - icon_dark = None - try: - builtin_tool = next( - provider - for provider in BuiltinToolManageService.list_builtin_tools( - user_id, - tenant_id, - ) - if provider.name == dict_metadata["provider"] - ) - icon = builtin_tool.icon - icon_dark = builtin_tool.icon_dark - except StopIteration: - pass - + icon, icon_dark = self._provider_icons(dict_metadata["provider"], icon) dict_metadata["icon"] = icon dict_metadata["icon_dark"] = icon_dark message.message.metadata = dict_metadata diff --git a/api/core/workflow/nodes/agent_v2/__init__.py b/api/core/workflow/nodes/agent_v2/__init__.py index eee44344721542..baf5cb1d1e779b 100644 --- a/api/core/workflow/nodes/agent_v2/__init__.py +++ b/api/core/workflow/nodes/agent_v2/__init__.py @@ -1,4 +1,3 @@ -from .agent_node import DifyAgentNode from .entities import DifyAgentNodeData -__all__ = ["DifyAgentNode", "DifyAgentNodeData"] +__all__ = ["DifyAgentNodeData"] diff --git a/api/core/workflow/nodes/agent_v2/session_store.py b/api/core/workflow/nodes/agent_v2/session_store.py index 31cea5ab088c53..51fa009304bb86 100644 --- a/api/core/workflow/nodes/agent_v2/session_store.py +++ b/api/core/workflow/nodes/agent_v2/session_store.py @@ -18,6 +18,7 @@ AgentWorkspaceOwnerType, ) from models.workflow import WorkflowNodeExecutionModel +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.workspace_service import ( AgentWorkspaceNotFoundError, AgentWorkspaceService, @@ -236,7 +237,7 @@ def load_or_create_node_execution_session( if resolved_binding is None or resolved_binding.agent_id != scope.agent_id: raise AgentWorkspaceNotFoundError("Workflow node participant Binding is unavailable") binding = resolved_binding - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, @@ -327,7 +328,7 @@ def _resolve_or_create_binding( agent_id=scope.agent_id, ) if existing_binding is not None: - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( existing_binding, base_home_snapshot_id=home_snapshot_id, agent_config_version_id=scope.agent_config_snapshot_id, diff --git a/api/core/workflow/nodes/human_input/__init__.py b/api/core/workflow/nodes/human_input/__init__.py index 2e8f3a7036fe47..f262d24c8f6fa9 100644 --- a/api/core/workflow/nodes/human_input/__init__.py +++ b/api/core/workflow/nodes/human_input/__init__.py @@ -1,51 +1 @@ -from enums.human_input import ( - ButtonStyle, - FormInputType, - HumanInputFormKind, - HumanInputFormStatus, - TimeoutUnit, - ValueSourceType, -) -from models.human_input_entities import ( - FileInputConfig, - FileListInputConfig, - FormDefinition, - FormInputConfig, - HumanInputNodeData, - HumanInputSubmissionValidationError, - ParagraphInputConfig, - SelectInputConfig, - StringListSource, - StringSource, - UserActionConfig, - validate_human_input_submission, -) - -from .pause_reason import DifyHITLEventType, HumanInputRequired, PauseReason -from .session_binding import SessionBinding, default_session_binding - -__all__ = [ - "ButtonStyle", - "DifyHITLEventType", - "FileInputConfig", - "FileListInputConfig", - "FormDefinition", - "FormInputConfig", - "FormInputType", - "HumanInputFormKind", - "HumanInputFormStatus", - "HumanInputNodeData", - "HumanInputRequired", - "HumanInputSubmissionValidationError", - "ParagraphInputConfig", - "PauseReason", - "SelectInputConfig", - "SessionBinding", - "StringListSource", - "StringSource", - "TimeoutUnit", - "UserActionConfig", - "ValueSourceType", - "default_session_binding", - "validate_human_input_submission", -] +"""Workflow runtime adapters for the shared Human Input domain.""" diff --git a/api/core/workflow/nodes/trigger_webhook/__init__.py b/api/core/workflow/nodes/trigger_webhook/__init__.py index e41d290f6d8b58..e69de29bb2d1d6 100644 --- a/api/core/workflow/nodes/trigger_webhook/__init__.py +++ b/api/core/workflow/nodes/trigger_webhook/__init__.py @@ -1,3 +0,0 @@ -from .node import TriggerWebhookNode - -__all__ = ["TriggerWebhookNode"] diff --git a/api/dev/check_knowledge_fs_coverage.py b/api/dev/check_knowledge_fs_coverage.py index 0bde50a61746f7..ca6428d137a84b 100644 --- a/api/dev/check_knowledge_fs_coverage.py +++ b/api/dev/check_knowledge_fs_coverage.py @@ -19,6 +19,7 @@ from typing import TypedDict, cast WORKSPACE_ROOT = Path(__file__).resolve().parents[2] +EXPLICIT_KNOWLEDGE_FS_PRODUCTION_PATHS = frozenset({"api/services/agent/knowledge_spaces.py"}) NON_CORE_COVERAGE_ALLOWLIST = frozenset( { "api/dev/check_knowledge_fs_coverage.py", @@ -152,11 +153,10 @@ def is_core_coverage_path(path: str) -> bool: """Return whether a repository-relative path belongs to the focused aggregate.""" if not path.endswith(".py"): return False - if path in { + if path in EXPLICIT_KNOWLEDGE_FS_PRODUCTION_PATHS or path in { "api/commands/knowledge_fs.py", "api/configs/extra/knowledge_fs_config.py", "api/controllers/openapi/knowledge_fs.py", - "api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py", "api/extensions/ext_knowledge_fs_observability.py", "api/services/dataset_knowledge_fs_upgrade_file_lease.py", "api/services/dataset_knowledge_fs_upgrade_service.py", @@ -199,7 +199,7 @@ def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]: def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]: - """Mirror the workflow's dynamic KnowledgeFS filename discovery.""" + """Mirror dynamic filename discovery and include explicit cross-domain owners.""" api_root = workspace_root / "api" if not api_root.is_dir(): raise CoverageGateError(f"Dify API directory does not exist: {api_root}") @@ -211,7 +211,7 @@ def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, . child_directories[:] = [name for name in child_directories if name != "__pycache__"] for filename in filenames: path = (current_directory / filename).relative_to(workspace_root).as_posix() - if filename.endswith(".py") and "knowledge_fs" in path: + if filename.endswith(".py") and ("knowledge_fs" in path or path in EXPLICIT_KNOWLEDGE_FS_PRODUCTION_PATHS): paths.add(path) if not paths: raise CoverageGateError("Dify KnowledgeFS production target set is empty") diff --git a/api/events/app_event.py b/api/events/app_event.py index 2fba0028f91878..d684ce9b4d5996 100644 --- a/api/events/app_event.py +++ b/api/events/app_event.py @@ -6,9 +6,6 @@ # sender: app, kwargs: app_model_config app_model_config_was_updated = signal("app-model-config-was-updated") -# sender: app, kwargs: published_workflow -app_published_workflow_was_updated = signal("app-published-workflow-was-updated") - # sender: app, kwargs: synced_draft_workflow app_draft_workflow_was_synced = signal("app-draft-workflow-was-synced") diff --git a/api/events/event_handlers/__init__.py b/api/events/event_handlers/__init__.py index ab655bfb067e0f..eb8b68f46c9331 100644 --- a/api/events/event_handlers/__init__.py +++ b/api/events/event_handlers/__init__.py @@ -10,21 +10,10 @@ from .queue_default_plugin_install_when_tenant_created import ( handle as handle_queue_default_plugin_install_when_tenant_created, ) -from .sync_knowledge_fs_bindings_when_app_published_workflow_updated import ( - handle as handle_sync_knowledge_fs_bindings_when_app_published_workflow_updated, -) -from .sync_plugin_trigger_when_app_created import handle as handle_sync_plugin_trigger_when_app_created from .sync_webhook_when_app_created import handle as handle_sync_webhook_when_app_created -from .sync_workflow_schedule_when_app_published import handle as handle_sync_workflow_schedule_when_app_published from .update_app_dataset_join_when_app_model_config_updated import ( handle as handle_update_app_dataset_join_when_app_model_config_updated, ) -from .update_app_dataset_join_when_app_published_workflow_updated import ( - handle as handle_update_app_dataset_join_when_app_published_workflow_updated, -) -from .update_app_triggers_when_app_published_workflow_updated import ( - handle as handle_update_app_triggers_when_app_published_workflow_updated, -) # Consolidated handler replaces both deduct_quota_when_message_created and # update_provider_last_used_at_when_message_created @@ -39,12 +28,7 @@ "handle_delete_tool_parameters_cache_when_sync_draft_workflow", "handle_queue_credential_sync_when_tenant_created", "handle_queue_default_plugin_install_when_tenant_created", - "handle_sync_knowledge_fs_bindings_when_app_published_workflow_updated", - "handle_sync_plugin_trigger_when_app_created", "handle_sync_webhook_when_app_created", - "handle_sync_workflow_schedule_when_app_published", "handle_update_app_dataset_join_when_app_model_config_updated", - "handle_update_app_dataset_join_when_app_published_workflow_updated", - "handle_update_app_triggers_when_app_published_workflow_updated", "handle_update_provider_when_message_created", ] diff --git a/api/events/event_handlers/create_installed_app_when_app_created.py b/api/events/event_handlers/create_installed_app_when_app_created.py index 7b0358364d7f1f..2dce23593ad727 100644 --- a/api/events/event_handlers/create_installed_app_when_app_created.py +++ b/api/events/event_handlers/create_installed_app_when_app_created.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session from events.app_event import app_was_created -from services.app_creation_records import create_installed_app_record +from repositories.app.creation_records import create_installed_app_record @app_was_created.connect diff --git a/api/events/event_handlers/create_site_record_when_app_created.py b/api/events/event_handlers/create_site_record_when_app_created.py index 45eff122321b57..aa009718e1d2bf 100644 --- a/api/events/event_handlers/create_site_record_when_app_created.py +++ b/api/events/event_handlers/create_site_record_when_app_created.py @@ -1,7 +1,7 @@ from sqlalchemy.orm import Session from events.app_event import app_was_created -from services.app_creation_records import create_site_record +from repositories.app.creation_records import create_site_record @app_was_created.connect diff --git a/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py b/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py index 59c30ad0427d62..ccce4357542b72 100644 --- a/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py +++ b/api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py @@ -2,12 +2,13 @@ from core.tools.entities.tool_entities import ToolProviderType from core.tools.errors import ToolProviderNotFoundError -from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolParameterConfigurationManager from core.workflow.human_input_adapter import adapt_node_config_for_graph from events.app_event import app_draft_workflow_was_synced +from extensions.ext_application_services import application_services from graphon.nodes import BuiltinNodeTypes from graphon.nodes.tool.entities import ToolEntity +from services.tools.tool_manager import ToolManager logger = logging.getLogger(__name__) @@ -25,6 +26,8 @@ def handle(sender, **kwargs): tool_entity = ToolEntity.model_validate(adapted_node_data["data"]) provider_type = ToolProviderType(tool_entity.provider_type.value) tool_runtime = ToolManager.get_tool_runtime( + tool_providers=application_services().tools.tool_providers, + workflow_queries=application_services().tools.workflow_queries, provider_type=provider_type, provider_id=tool_entity.provider_id, tool_name=tool_entity.tool_name, diff --git a/api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py b/api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py deleted file mode 100644 index f6ca2271c03d82..00000000000000 --- a/api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from typing import cast - -from sqlalchemy import select -from sqlalchemy.orm import Session - -from core.db.session_factory import session_factory -from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from models.knowledge_fs import AppKnowledgeFSSpaceJoin, KnowledgeFSAppSpaceJoinType -from models.model import App -from models.workflow import Workflow -from services.agent.knowledge_spaces import collect_workflow_agent_knowledge_space_ids -from services.knowledge_fs.knowledge_retrieval_v2.validation import collect_control_space_ids -from services.knowledge_fs.runtime import get_knowledge_fs_runtime - - -@app_published_workflow_was_updated.connect -def handle(sender: object, **kwargs: object) -> None: - app = cast(App, sender) - published_workflow = cast(Workflow, kwargs["published_workflow"]) - publish_session = cast(Session | None, kwargs.get("session")) - control_space_ids = get_control_space_ids_from_workflow(published_workflow) - - # Agent bindings are copied to the published Workflow before this signal. - # Flush them so the union cannot silently omit an Agent node in this transaction. - graph = published_workflow.graph_dict - if isinstance(graph, dict) and any( - isinstance(node, dict) and isinstance(node.get("data"), dict) and is_dify_agent_node_data(node["data"]) - for node in graph.get("nodes", []) - ): - lookup_session = publish_session if publish_session is not None else cast(Session, db.session) - lookup_session.flush() - control_space_ids = tuple( - dict.fromkeys( - ( - *control_space_ids, - *collect_workflow_agent_knowledge_space_ids(session=lookup_session, workflow=published_workflow), - ) - ) - ) - - if not control_space_ids: - lookup_session = publish_session if publish_session is not None else cast(Session, db.session) - existing_binding = lookup_session.scalar( - select(AppKnowledgeFSSpaceJoin.id) - .where( - AppKnowledgeFSSpaceJoin.tenant_id == app.tenant_id, - AppKnowledgeFSSpaceJoin.app_id == app.id, - AppKnowledgeFSSpaceJoin.join_type == KnowledgeFSAppSpaceJoinType.WORKFLOW, - ) - .limit(1) - ) - if existing_binding is None: - return - - runtime = get_knowledge_fs_runtime(session_factory.get_session_maker()) - if publish_session is None: - runtime.app_bindings.sync_workflow_bindings( - tenant_id=app.tenant_id, - actor_account_id=published_workflow.created_by, - app_id=app.id, - control_space_ids=list(control_space_ids), - ) - else: - runtime.app_bindings.sync_workflow_bindings( - tenant_id=app.tenant_id, - actor_account_id=published_workflow.created_by, - app_id=app.id, - control_space_ids=list(control_space_ids), - session=publish_session, - ) - - -def get_control_space_ids_from_workflow(published_workflow: Workflow) -> tuple[str, ...]: - graph = published_workflow.graph_dict - return collect_control_space_ids(graph) if isinstance(graph, dict) else () - - -__all__ = ["get_control_space_ids_from_workflow", "handle"] diff --git a/api/events/event_handlers/sync_plugin_trigger_when_app_created.py b/api/events/event_handlers/sync_plugin_trigger_when_app_created.py deleted file mode 100644 index 19b8b263faaefe..00000000000000 --- a/api/events/event_handlers/sync_plugin_trigger_when_app_created.py +++ /dev/null @@ -1,31 +0,0 @@ -import logging - -from events.app_event import app_published_workflow_was_updated -from models.model import App, AppMode -from models.workflow import Workflow -from services.trigger.trigger_service import TriggerService - -logger = logging.getLogger(__name__) - - -@app_published_workflow_was_updated.connect -def handle( - sender, - published_workflow: Workflow | None = None, - **kwargs, -): - """ - Sync plugin trigger relationships when a workflow is published. - - Production trigger dispatch relies on these relationships; debug dispatch - uses the workflow graph directly. - """ - app: App = sender - if app.mode != AppMode.WORKFLOW.value: - # only handle workflow app, chatflow is not supported yet - return - - if published_workflow is None: - return - - TriggerService.sync_plugin_trigger_relationships(app, published_workflow) diff --git a/api/events/event_handlers/sync_webhook_when_app_created.py b/api/events/event_handlers/sync_webhook_when_app_created.py index 7fd23a8dfa4176..e34a7bc14bed5d 100644 --- a/api/events/event_handlers/sync_webhook_when_app_created.py +++ b/api/events/event_handlers/sync_webhook_when_app_created.py @@ -1,6 +1,8 @@ import logging -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated +from sqlalchemy.orm import Session, sessionmaker + +from events.app_event import app_draft_workflow_was_synced from models.model import App, AppMode from models.workflow import Workflow from services.trigger.webhook_service import WebhookService @@ -9,7 +11,13 @@ @app_draft_workflow_was_synced.connect -def handle(sender, synced_draft_workflow: Workflow, **kwargs): +def handle( + sender, + synced_draft_workflow: Workflow, + *, + session_factory: sessionmaker[Session] | None = None, + **kwargs, +): """ While creating a workflow or updating a workflow, we may need to sync its webhook relationships in DB. @@ -19,14 +27,6 @@ def handle(sender, synced_draft_workflow: Workflow, **kwargs): # only handle workflow app, chatflow is not supported yet return - WebhookService.sync_webhook_relationships(app, synced_draft_workflow, remove_stale=False) - - -@app_published_workflow_was_updated.connect -def handle_published(sender, published_workflow: Workflow, **kwargs): - """Remove stale webhook relationships after their node deletion is published.""" - app: App = sender - if app.mode != AppMode.WORKFLOW.value: - return - - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) + WebhookService.sync_webhook_relationships( + app, synced_draft_workflow, remove_stale=False, session_factory=session_factory + ) diff --git a/api/events/event_handlers/sync_workflow_schedule_when_app_published.py b/api/events/event_handlers/sync_workflow_schedule_when_app_published.py deleted file mode 100644 index 5f8fcd86170b7a..00000000000000 --- a/api/events/event_handlers/sync_workflow_schedule_when_app_published.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from typing import cast - -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker - -from core.workflow.nodes.trigger_schedule.entities import SchedulePlanUpdate -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from models import AppMode, Workflow, WorkflowSchedulePlan -from services.trigger.schedule_service import ScheduleService - -logger = logging.getLogger(__name__) - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - """ - Handle app published workflow update event to sync workflow_schedule_plans table. - - When a workflow is published, this handler will: - 1. Extract schedule trigger nodes from the workflow graph - 2. Compare with existing workflow_schedule_plans records - 3. Create/update/delete schedule plans as needed - """ - app = sender - if app.mode != AppMode.WORKFLOW.value: - return - - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - - sync_schedule_from_workflow(tenant_id=app.tenant_id, app_id=app.id, workflow=published_workflow) - - -def sync_schedule_from_workflow(tenant_id: str, app_id: str, workflow: Workflow) -> WorkflowSchedulePlan | None: - """ - Sync schedule plan from workflow graph configuration. - - Args: - tenant_id: Tenant ID - app_id: App ID - workflow: Published workflow instance - - Returns: - Updated or created WorkflowSchedulePlan, or None if no schedule node - """ - with sessionmaker(db.engine).begin() as session: - schedule_config = ScheduleService.extract_schedule_config(workflow) - - existing_plan = session.scalar( - select(WorkflowSchedulePlan).where( - WorkflowSchedulePlan.tenant_id == tenant_id, - WorkflowSchedulePlan.app_id == app_id, - ) - ) - - if not schedule_config: - if existing_plan: - logger.info("No schedule node in workflow for app %s, removing schedule plan", app_id) - ScheduleService.delete_schedule(session=session, schedule_id=existing_plan.id) - return None - - if existing_plan: - updates = SchedulePlanUpdate( - node_id=schedule_config.node_id, - cron_expression=schedule_config.cron_expression, - timezone=schedule_config.timezone, - ) - updated_plan = ScheduleService.update_schedule( - session=session, - schedule_id=existing_plan.id, - updates=updates, - ) - return updated_plan - else: - new_plan = ScheduleService.create_schedule( - session=session, - tenant_id=tenant_id, - app_id=app_id, - config=schedule_config, - ) - return new_plan diff --git a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py b/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py deleted file mode 100644 index 6769b94cde0093..00000000000000 --- a/api/events/event_handlers/update_app_dataset_join_when_app_published_workflow_updated.py +++ /dev/null @@ -1,69 +0,0 @@ -from typing import cast - -from sqlalchemy import delete, select - -from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from graphon.nodes import BuiltinNodeTypes -from models.dataset import AppDatasetJoin -from models.workflow import Workflow - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - app = sender - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - - dataset_ids = get_dataset_ids_from_workflow(published_workflow) - app_dataset_joins = db.session.scalars(select(AppDatasetJoin).where(AppDatasetJoin.app_id == app.id)).all() - - removed_dataset_ids: set[str] = set() - if not app_dataset_joins: - added_dataset_ids = dataset_ids - else: - old_dataset_ids: set[str] = set() - old_dataset_ids.update(app_dataset_join.dataset_id for app_dataset_join in app_dataset_joins) - - added_dataset_ids = dataset_ids - old_dataset_ids - removed_dataset_ids = old_dataset_ids - dataset_ids - - if removed_dataset_ids: - for dataset_id in removed_dataset_ids: - db.session.execute( - delete(AppDatasetJoin).where(AppDatasetJoin.app_id == app.id, AppDatasetJoin.dataset_id == dataset_id) - ) - - if added_dataset_ids: - for dataset_id in added_dataset_ids: - app_dataset_join = AppDatasetJoin(app_id=app.id, dataset_id=dataset_id) - db.session.add(app_dataset_join) - - db.session.commit() - - -def get_dataset_ids_from_workflow(published_workflow: Workflow) -> set[str]: - dataset_ids: set[str] = set() - graph = published_workflow.graph_dict - if not graph: - return dataset_ids - - nodes = graph.get("nodes", []) - - # fetch all knowledge retrieval nodes - knowledge_retrieval_nodes = [ - node for node in nodes if node.get("data", {}).get("type") == BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL - ] - - if not knowledge_retrieval_nodes: - return dataset_ids - - for node in knowledge_retrieval_nodes: - try: - node_data = KnowledgeRetrievalNodeData.model_validate(node.get("data", {})) - dataset_ids.update(dataset_id for dataset_id in node_data.dataset_ids) - except Exception: - continue - - return dataset_ids diff --git a/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py b/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py deleted file mode 100644 index d55fe262fb25a3..00000000000000 --- a/api/events/event_handlers/update_app_triggers_when_app_published_workflow_updated.py +++ /dev/null @@ -1,112 +0,0 @@ -from typing import cast - -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker - -from core.trigger.constants import TRIGGER_NODE_TYPES -from events.app_event import app_published_workflow_was_updated -from extensions.ext_database import db -from models import AppMode -from models.enums import AppTriggerStatus -from models.trigger import AppTrigger -from models.workflow import Workflow - - -@app_published_workflow_was_updated.connect -def handle(sender, **kwargs): - """ - Handle app published workflow update event to sync app_triggers table. - - When a workflow is published, this handler will: - 1. Extract trigger nodes from the workflow graph - 2. Compare with existing app_triggers records - 3. Add new triggers and remove obsolete ones - """ - app = sender - if app.mode != AppMode.WORKFLOW.value: - return - - published_workflow = kwargs.get("published_workflow") - published_workflow = cast(Workflow, published_workflow) - # Extract trigger info from workflow - trigger_infos = get_trigger_infos_from_workflow(published_workflow) - - with sessionmaker(db.engine).begin() as session: - # Get existing app triggers - existing_triggers = ( - session.execute( - select(AppTrigger).where(AppTrigger.tenant_id == app.tenant_id, AppTrigger.app_id == app.id) - ) - .scalars() - .all() - ) - - # Convert existing triggers to dict for easy lookup - existing_triggers_map = {trigger.node_id: trigger for trigger in existing_triggers} - - # Get current and new node IDs - existing_node_ids = set(existing_triggers_map.keys()) - new_node_ids = {info["node_id"] for info in trigger_infos} - - # Calculate changes - added_node_ids = new_node_ids - existing_node_ids - removed_node_ids = existing_node_ids - new_node_ids - - # Remove obsolete triggers - for node_id in removed_node_ids: - session.delete(existing_triggers_map[node_id]) - - for trigger_info in trigger_infos: - node_id = trigger_info["node_id"] - - if node_id in added_node_ids: - # Create new trigger - app_trigger = AppTrigger( - tenant_id=app.tenant_id, - app_id=app.id, - trigger_type=trigger_info["node_type"], - title=trigger_info["node_title"], - node_id=node_id, - provider_name=trigger_info.get("node_provider_name", ""), - status=AppTriggerStatus.ENABLED, - ) - session.add(app_trigger) - elif node_id in existing_node_ids: - # Update existing trigger if needed - existing_trigger = existing_triggers_map[node_id] - new_title = trigger_info["node_title"] - if new_title and existing_trigger.title != new_title: - existing_trigger.title = new_title - session.add(existing_trigger) - - -def get_trigger_infos_from_workflow(published_workflow: Workflow) -> list[dict]: - """ - Extract trigger node information from the workflow graph. - - Returns: - List of trigger info dictionaries containing: - - node_type: The type of the trigger node ('trigger-webhook', 'trigger-schedule', 'trigger-plugin') - - node_id: The node ID in the workflow - - node_title: The title of the node - - node_provider_name: The name of the node's provider, only for plugin - """ - graph = published_workflow.graph_dict - if not graph: - return [] - - nodes = graph.get("nodes", []) - trigger_types = TRIGGER_NODE_TYPES - - trigger_infos = [ - { - "node_type": node.get("data", {}).get("type"), - "node_id": node.get("id"), - "node_title": node.get("data", {}).get("title"), - "node_provider_name": node.get("data", {}).get("provider_name"), - } - for node in nodes - if node.get("data", {}).get("type") in trigger_types - ] - - return trigger_infos diff --git a/api/extensions/application_services/agent.py b/api/extensions/application_services/agent.py index d88ddff4b3a8fc..f94025df593c79 100644 --- a/api/extensions/application_services/agent.py +++ b/api/extensions/application_services/agent.py @@ -7,29 +7,36 @@ from configs import dify_config from core.app.file_access import DatabaseFileAccessController +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.app.agent_app_repository import AgentAppRepository from repositories.app.agent_sandbox_repository import AgentSandboxRepository from repositories.app.generation_repository import AppGenerationRepository from repositories.tools.provider_repository import ToolProviderRepository from services.agent.log_file_gateway import AgentLogFileGateway from services.agent.log_service import AgentLogService +from services.agent.tool_invocation_service import AgentToolInnerService from services.app.agent_app_feature_gateway import AgentAppFeatureValidator from services.app.agent_app_sandbox_service import AgentAppSandboxService from services.app.agent_app_service import AgentAppAccessService, AgentAppFeatureConfigService from services.app.agent_sandbox_file_gateway import AgentSandboxFileGateway, create_sandbox_client from services.file_request_service import FileRequestService +from services.tools.agent_invocation_gateway import AgentToolInvocationGateway from services.tools.tool_manager import ToolManager +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) class AgentAppServices: - logs: AgentLogService access: AgentAppAccessService features: AgentAppFeatureConfigService sandbox: AgentAppSandboxService + tools: AgentToolInnerService + logs: AgentLogService -def build_agent_app_services(*, database_client: sessionmaker[Session]) -> AgentAppServices: +def build_agent_app_services( + *, database_client: sessionmaker[Session], variables: WorkflowExecutionVariables +) -> AgentAppServices: apps = AgentAppRepository(session_factory=database_client) return AgentAppServices( logs=AgentLogService( @@ -37,6 +44,12 @@ def build_agent_app_services(*, database_client: sessionmaker[Session]) -> Agent icons=partial(ToolManager.get_tool_icon, tool_providers=ToolProviderRepository(database_client)), files=AgentLogFileGateway(sessions=database_client, access=DatabaseFileAccessController()), ), + tools=AgentToolInnerService( + apps=apps, + tools=AgentToolInvocationGateway( + variables=variables, runtime=build_workflow_execution_dependencies(database_client) + ), + ), access=AgentAppAccessService(references=apps), features=AgentAppFeatureConfigService(features=apps, validator=AgentAppFeatureValidator()), sandbox=AgentAppSandboxService( diff --git a/api/extensions/application_services/app.py b/api/extensions/application_services/app.py index 14f10db69a463c..1931ee06baceb9 100644 --- a/api/extensions/application_services/app.py +++ b/api/extensions/application_services/app.py @@ -1,11 +1,16 @@ """Composition of App use cases.""" +from __future__ import annotations + from dataclasses import dataclass +from typing import TYPE_CHECKING from sqlalchemy.orm import Session, sessionmaker from configs import dify_config +from extensions.application_services.workflow import build_app_dsl_service from libs.datetime_utils import naive_utc_now +from models.tool_runtime_contracts import WorkflowToolStore from repositories.account.repository import SQLAlchemyAccountRepository from repositories.app.api_key_repository import AppApiKeyRepository from repositories.app.console_repository import ConsoleAppRepository @@ -33,9 +38,13 @@ from services.oauth_server_service import OAuthServerService from services.rbac.members import MemberService from services.recommended_app_package_service import RecommendedAppPackageService +from services.tools.provider_queries import ToolProviders from services.trigger.app_trigger_store import AppTriggerStore from services.workflow.dsl_import import DslImportService +if TYPE_CHECKING: + from services.data_migration.import_service import WorkflowPublisher + @dataclass(frozen=True, slots=True) class AppServices: @@ -52,6 +61,9 @@ def build_app_services( *, database_client: sessionmaker[Session], oauth: OAuthServerService, + tool_providers: ToolProviders, + workflow_tools: WorkflowToolStore, + workflow_publisher: WorkflowPublisher, recommended_packages: RecommendedAppPackageService, repository: ConsoleAppRepository, rbac_members: MemberService, @@ -59,11 +71,13 @@ def build_app_services( ) -> AppServices: transfers = AppTransferGateway( session_factory=database_client, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), recommended_packages=recommended_packages, + workflow_tools=workflow_tools, + workflow_publisher=workflow_publisher, ) return AppServices( imports=AppImportService(accounts=SQLAlchemyAccountRepository(database_client), definitions=transfers), @@ -81,8 +95,10 @@ def build_app_services( access=EnterpriseConsoleAppAccess(members=rbac_members), transfers=transfers, creators=CreatorsPlatformGateway(oauth=oauth), + lifecycle=AppLifecycleGateway( + session_factory=database_client, tool_providers=tool_providers, workflow_queries=workflow_tools + ), tracing=tracing_configs, - lifecycle=AppLifecycleGateway(session_factory=database_client), ), enterprise=EnterpriseAppService( apps=EnterpriseAppRepository( diff --git a/api/extensions/application_services/installed_app.py b/api/extensions/application_services/installed_app.py index afe9418ddb2e77..e4ae6fb8e53667 100644 --- a/api/extensions/application_services/installed_app.py +++ b/api/extensions/application_services/installed_app.py @@ -4,6 +4,7 @@ from sqlalchemy.orm import Session, sessionmaker +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.conversation_command_service import ConversationCommandService from services.conversation_name_service import ConversationNameService @@ -14,6 +15,7 @@ from services.installed_app_generation_service import InstalledAppGenerationService from services.installed_app_service import InstalledAppService, WorkspaceRoleLookup from services.webapp_access_query_service import WebAppAccessQueryService +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) @@ -33,6 +35,7 @@ def build_installed_app_services( webapp_access: WebAppAccessQueryService, get_workspace_role: WorkspaceRoleLookup, webapp_auth_enabled: bool, + variables: WorkflowExecutionVariables, ) -> InstalledAppServices: installed_apps = SQLAlchemyInstalledAppRepository(session_factory=database_client) access = InstalledAppAccessService( @@ -50,7 +53,11 @@ def build_installed_app_services( ), generation=InstalledAppGenerationService( usage=installed_apps, - runtime=AppGenerateServiceRuntime(session_factory=database_client), + runtime=AppGenerateServiceRuntime( + session_factory=database_client, + variables=variables, + runtime=build_workflow_execution_dependencies(database_client), + ), ), conversations=InstalledAppConversationService( queries=conversation_queries, diff --git a/api/extensions/application_services/knowledge.py b/api/extensions/application_services/knowledge.py index 3e0f98412269b0..9326143f755f43 100644 --- a/api/extensions/application_services/knowledge.py +++ b/api/extensions/application_services/knowledge.py @@ -1,16 +1,19 @@ """Composition root for dataset-controller application services.""" from dataclasses import dataclass +from functools import partial from uuid import uuid4 from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.rag.extractor.entity.datasource_type import DatasourceType +from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.retrieval import build_dataset_retrieval +from extensions.application_services.workflow import build_workflow_execution_dependencies from libs.helper import generate_text_hash from repositories.app.console_repository import ConsoleAppRepository +from repositories.factory import DifyAPIRepositoryFactory from repositories.knowledge.dataset_api_key_repository import DatasetApiKeyRepository from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository @@ -18,6 +21,8 @@ from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository +from repositories.workflow.pipeline_publication_repository import PipelinePublicationRepository from services.api_token_service import ApiTokenCache, get_effective_token_last_used_at from services.data_source.credential_gateway import ( ActorAwareDatasourceCredentialGateway, @@ -55,9 +60,13 @@ from services.knowledge.segments.indexing import SegmentIndexingGateway from services.knowledge_fs.feature_service import get_knowledge_fs_features from services.knowledge_retrieval_inner_service import InnerKnowledgeRetrievalService +from services.rag_pipeline.execution_service import RagPipelineExecutionService +from services.rag_pipeline.publication_gateway import PipelineIndexUpdateGateway, PipelinePublicationModelGateway +from services.rag_pipeline.publication_service import PipelinePublicationService from services.rbac.members import MemberService from services.tag_application_service import TagTargetQuery -from services.workflow_run_agg import WorkflowRunAgg +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task from tasks.delete_segment_from_index_task import delete_segment_from_index_task from tasks.disable_segments_from_index_task import disable_segments_from_index_task @@ -76,6 +85,8 @@ class KnowledgeServices: indexing_estimates: IndexingEstimateApplicationService segments: DatasetSegmentApplicationService pipeline_generator: PipelineGenerator + pipeline_execution: RagPipelineExecutionService + pipeline_publication: PipelinePublicationService upgrades: KnowledgeUpgradeApplicationService @@ -102,6 +113,7 @@ def build_knowledge_services( providers: DatasourceProviderService, redis: RedisSegmentClient, tags: TagTargetQuery, + variables: WorkflowExecutionVariables, rbac_members: MemberService, ) -> KnowledgeServices: """Build the dataset-controller knowledge use cases.""" @@ -130,6 +142,7 @@ def build_knowledge_services( enable_task=enable_segments_to_index_task.delay, disable_task=disable_segments_from_index_task.delay, ) + workflow_runtime = build_workflow_execution_dependencies(database_client) dataset_operations = SQLAlchemyDatasetOperations( session_factory=database_client, tags=tags, @@ -161,7 +174,27 @@ def build_knowledge_services( metadata_schema=DocumentService.DOCUMENT_METADATA_SCHEMA, ), pipeline_generator=PipelineGenerator( - execution_driver=WorkflowRunAgg.run, documents=documents, datasource_providers=providers + runtime=workflow_runtime, + execution_driver=workflow_runtime.execution_driver, + documents=documents, + datasource_providers=providers, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ), + pipeline_publication=PipelinePublicationService( + PipelinePublicationRepository(database_client), + PipelinePublicationModelGateway(), + PipelineIndexUpdateGateway(), + ), + pipeline_execution=RagPipelineExecutionService( + runtime=workflow_runtime, + workflows=WorkflowDefinitionRepository(session_factory=database_client), + variables=variables, + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(database_client), + writer_factory=partial( + DifyCoreRepositoryFactory.create_workflow_node_execution_repository, session_factory=database_client + ), + documents=documents, ), upgrades=KnowledgeUpgradeApplicationService( session_factory=database_client, diff --git a/api/extensions/application_services/snippets.py b/api/extensions/application_services/snippets.py new file mode 100644 index 00000000000000..b7a4021e31f138 --- /dev/null +++ b/api/extensions/application_services/snippets.py @@ -0,0 +1,65 @@ +"""Composition of Snippet use cases and their workflow Agent dependencies.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy.orm import Session, sessionmaker + +from extensions.application_services.agent_bindings import build_workflow_agent_service +from extensions.application_services.workflow import build_workflow_execution_dependencies +from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader +from services.skill_management_service import SkillManagementService +from services.workflow.variable_contracts import WorkflowExecutionVariables + +if TYPE_CHECKING: + from services.snippet_dsl_service import SnippetDslService + from services.snippet_generate_service import SnippetGenerateService + from services.snippet_service import SnippetService + + +def build_snippet_generation_service( + *, database_client: sessionmaker[Session], variables: WorkflowExecutionVariables +) -> SnippetGenerateService: + from services.snippet_generate_service import SnippetGenerateService + from services.workflow_service import WorkflowService + + return SnippetGenerateService( + runtime=build_workflow_execution_dependencies(database_client), + snippets=build_snippet_service(database_client), + variables=variables, + workflows=WorkflowService(database_client, runtime=build_workflow_execution_dependencies(database_client)), + ) + + +def build_snippet_service( + session_maker: sessionmaker[Session] | Session | None = None, session: Session | None = None +) -> SnippetService: + from services.snippet_service import SnippetService + + if isinstance(session_maker, Session): + session = session_maker + session_maker = None + if session is not None: + session_maker = sessionmaker(bind=session.get_bind(), expire_on_commit=False) + if session_maker is None: + raise ValueError("SnippetService requires a session or session_maker.") + return SnippetService( + session_maker=session_maker, + session=session, + agent_bindings=build_workflow_agent_service, + skills=WorkflowAgentSkillReader(SkillManagementService(session_maker=session_maker)), + ) + + +def build_snippet_dsl_service(session: Session) -> SnippetDslService: + from extensions.application_services.workflow import build_workflow_drafts + from services.snippet_dsl_service import SnippetDslService + + return SnippetDslService( + session, + snippets=build_snippet_service(session=session), + drafts=build_workflow_drafts( + sessionmaker(bind=session.get_bind(), expire_on_commit=False), import_session=session + ), + ) diff --git a/api/extensions/application_services/trial_app.py b/api/extensions/application_services/trial_app.py index be28822cf8dbd1..12cf05036a4e5f 100644 --- a/api/extensions/application_services/trial_app.py +++ b/api/extensions/application_services/trial_app.py @@ -4,11 +4,13 @@ from sqlalchemy.orm import Session, sessionmaker +from extensions.application_services.workflow import build_workflow_execution_dependencies from repositories.trial_app_repository import TrialAppRepository from services.trial_app_access_service import TrialAppAccessService from services.trial_app_generation_adapters import AppGenerateServiceRuntime from services.trial_app_generation_service import TrialAppGenerationService from services.trial_app_usage import TrialAppUsageRecorder +from services.workflow.variable_contracts import WorkflowExecutionVariables @dataclass(frozen=True, slots=True) @@ -19,12 +21,17 @@ class TrialAppServices: def build_trial_app_services( - *, database_client: sessionmaker[Session], trial_apps: TrialAppRepository + *, database_client: sessionmaker[Session], trial_apps: TrialAppRepository, variables: WorkflowExecutionVariables ) -> TrialAppServices: return TrialAppServices( access=TrialAppAccessService(apps=trial_apps), generation=TrialAppGenerationService( - runtime=AppGenerateServiceRuntime(session_factory=database_client), usage=trial_apps + runtime=AppGenerateServiceRuntime( + session_factory=database_client, + variables=variables, + runtime=build_workflow_execution_dependencies(database_client), + ), + usage=trial_apps, ), usage=trial_apps, ) diff --git a/api/extensions/application_services/workflow.py b/api/extensions/application_services/workflow.py index 985283b8de7532..285a45187ebfd7 100644 --- a/api/extensions/application_services/workflow.py +++ b/api/extensions/application_services/workflow.py @@ -8,18 +8,25 @@ from sqlalchemy.orm import Session, sessionmaker +from core.app.entities.app_invoke_entities import InvokeFrom +from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler +from core.ops.ops_trace_manager import TraceQueueManager from core.tools.__base.tool import Tool -from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolInvokeMeta +from extensions.ext_redis import RedisClientWrapper from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings from models.annotation_reply import AnnotationReplies from models.human_input_contracts import HumanInputFormFactory +from models.model import Message from models.tool_runtime_contracts import WorkflowToolQueries from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository -from services.agent.chat.ports import AgentDatasetTools +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.agent_config import AgentAppConfigurations +from services.app.generation.ports import MessageFileWriter from services.human_input.ports import HumanInputFormReader from services.knowledge.retrieval.ports import DatasetRetrievalFactory +from services.rbac.members import MemberService from services.tools.provider_queries import ToolProviders from services.tools.tool_engine import ToolEngine from services.workflow.execution.chatflow_ports import ChatflowRecords, ConversationVariables @@ -33,6 +40,7 @@ WorkflowExecutionLogs, WorkflowToolInvoker, ) +from services.workflow.variable_contracts import WorkflowExecutionVariables if TYPE_CHECKING: from core.app.apps.workflow_app_runner import WorkflowRunDriver @@ -71,6 +79,47 @@ def __call__( ) +class _SessionBoundAgentToolInvoker(AgentToolInvoker): + """Own one short-lived Session for each eager Agent tool invocation.""" + + def __init__(self, sessions: sessionmaker[Session]) -> None: + self._sessions = sessions + + @override + def __call__( + self, + tool: Tool, + tool_parameters: str | dict[str, Any], + user_id: str, + tenant_id: str, + message: Message, + invoke_from: InvokeFrom, + agent_tool_callback: DifyAgentCallbackHandler, + trace_manager: TraceQueueManager | None = None, + conversation_id: str | None = None, + app_id: str | None = None, + message_id: str | None = None, + *, + records: MessageFileWriter, + ) -> tuple[str, list[str], ToolInvokeMeta]: + with self._sessions() as session: + return ToolEngine.agent_invoke( + session=session, + tool=tool, + tool_parameters=tool_parameters, + user_id=user_id, + tenant_id=tenant_id, + message=message, + invoke_from=invoke_from, + agent_tool_callback=agent_tool_callback, + trace_manager=trace_manager, + conversation_id=conversation_id, + app_id=app_id, + message_id=message_id, + records=records, + ) + + @dataclass(frozen=True) class WorkflowExecutionDependencies: execution_driver: WorkflowRunDriver @@ -82,6 +131,7 @@ class WorkflowExecutionDependencies: tools: WorkflowToolQueries tool_providers: ToolProviders tool_invoker: WorkflowToolInvoker + agent_tool_invoker: AgentToolInvoker agent_bindings: WorkflowAgentRuntimeBindings chat_records: ChatflowRecords annotation_replies: AnnotationReplies @@ -144,6 +194,7 @@ def build_workflow_execution_dependencies(database_client: sessionmaker[Session] tools=WorkflowToolRepository(database_client), tool_providers=ToolProviderRepository(database_client), tool_invoker=_SessionBoundWorkflowToolInvoker(database_client), + agent_tool_invoker=_SessionBoundAgentToolInvoker(database_client), agent_bindings=WorkflowAgentBindingResolver(database_client), chat_records=records, annotation_replies=build_annotation_replies(database_client), @@ -161,6 +212,153 @@ def build_workflow_execution_dependencies(database_client: sessionmaker[Session] ) +if TYPE_CHECKING: + from services.agent.workflow_contracts import WorkflowAgentBindingStore + from services.app_dsl_service import AppDslService + from services.workflow.console_service import ConsoleWorkflowService + from services.workflow.definition_gateway import ( + WorkflowDefinitionGateway, + WorkflowDefinitionReader, + WorkflowDraftReader, + ) + from services.workflow.draft_service import DraftDefinitions, DraftLifecycle, WorkflowDraftService + from services.workflow_service import WorkflowService + + +def build_console_workflow_service( + *, + database_client: sessionmaker[Session], + redis: RedisClientWrapper, + variables: WorkflowExecutionVariables, + rbac_members: MemberService, +) -> ConsoleWorkflowService[WorkflowAgentBindingStore]: + from core.app.file_access import DatabaseFileAccessController + from core.helper.encrypter import decrypt_token + from repositories.app.console_repository import ConsoleAppRepository + from repositories.workflow.collaboration_repository import WorkflowCollaborationRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from repositories.workflow.draft_repository import WorkflowDraftRepository + from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository + from services.agent.workflow_publish_service import WorkflowAgentPublishService + from services.app.console_gateway import EnterpriseConsoleAppAccess + from services.app_generate_service import AppGenerateService + from services.app_service import AppService + from services.app_task_service import AppTaskControlService + from services.workflow.access_gateway import WorkflowAccessGateway + from services.workflow.console_service import ConsoleWorkflowService + from services.workflow.conversion_service import WorkflowConversionService + from services.workflow.runtime_gateway import WorkflowRuntimeGateway + from services.workflow.workflow_converter import WorkflowConverter + from services.workflow_service import WorkflowService + + workflows = WorkflowService( + session_maker=database_client, runtime=build_workflow_execution_dependencies(database_client) + ) + + definitions = WorkflowDefinitionRepository(session_factory=database_client) + drafts = WorkflowDraftRepository(database_client) + lifecycle = build_workflow_definition_gateway(database_client, definitions, workflows, drafts=drafts) + apps = ConsoleAppRepository(session_factory=database_client) + return ConsoleWorkflowService( + conversion=WorkflowConversionService( + apps, + converter=WorkflowConverter(), + decrypt_token=decrypt_token, + notify_created=AppService.notify_created_app, + ), + agent_services=WorkflowAgentPublishService, + definitions=definitions, + drafts=build_workflow_draft_service(drafts, lifecycle, database_client), + lifecycle=lifecycle, + runtime=WorkflowRuntimeGateway( + runtime=build_workflow_execution_dependencies(database_client), + variables=variables, + session_factory=database_client, + workflows=workflows, + definitions=definitions, + reservations=WorkflowDebugReservationRepository(database_client), + executions=WorkflowNodeExecutionRepository(database_client), + generator=AppGenerateService, + tasks=AppTaskControlService(redis_client=redis), + file_access=DatabaseFileAccessController(), + ), + apps=apps, + presence=WorkflowCollaborationRepository(redis=redis), + access=WorkflowAccessGateway( + session_factory=database_client, apps=EnterpriseConsoleAppAccess(members=rbac_members) + ), + ) + + +def build_app_dsl_service(session: Session) -> AppDslService: + from repositories.app.dsl_repository import AppDslOverwriteRepository + from services.app_dsl_service import AppDslService + + sessions = sessionmaker(bind=session.get_bind(), expire_on_commit=False) + return AppDslService( + session, + drafts=build_workflow_drafts(sessions, import_session=session), + overwrites=AppDslOverwriteRepository(sessions=sessions, import_session=session), + ) + + +def build_workflow_definition_gateway( + database_client: sessionmaker[Session], + definitions: WorkflowDefinitionReader, + workflows: WorkflowService, + *, + drafts: WorkflowDraftReader, +) -> WorkflowDefinitionGateway: + from repositories.credentials.query_repository import CredentialQueryRepository + from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader + from services.skill_management_service import SkillManagementService + from services.workflow.definition_gateway import WorkflowDefinitionGateway + + return WorkflowDefinitionGateway( + session_factory=database_client, + definitions=definitions, + drafts=drafts, + workflows=workflows, + credentials=CredentialQueryRepository(session_factory=database_client), + skills=WorkflowAgentSkillReader(SkillManagementService(session_maker=database_client)), + ) + + +def build_workflow_draft_service( + definitions: DraftDefinitions[WorkflowAgentBindingStore], lifecycle: DraftLifecycle, sessions: sessionmaker[Session] +) -> WorkflowDraftService[WorkflowAgentBindingStore]: + from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.agent.workflow_publish_service import WorkflowAgentPublishService + from services.workflow.draft_service import WorkflowDraftService + + return WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=definitions, + lifecycle=lifecycle, + retirement=WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sessions)), + ) + + +def build_workflow_drafts( + database_client: sessionmaker[Session], *, import_session: Session | None = None +) -> WorkflowDraftService[WorkflowAgentBindingStore]: + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from repositories.workflow.draft_repository import WorkflowDraftRepository + from services.workflow_service import WorkflowService + + definitions = WorkflowDefinitionRepository(session_factory=database_client, import_session=import_session) + drafts = WorkflowDraftRepository(database_client, import_session=import_session) + lifecycle = build_workflow_definition_gateway( + database_client, + definitions, + WorkflowService(database_client, runtime=build_workflow_execution_dependencies(database_client)), + drafts=drafts, + ) + return build_workflow_draft_service(drafts, lifecycle, database_client) + + def build_workflow_suggestions(database_client: sessionmaker[Session]): from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository from services.workflow.generation.suggestions import WorkflowInstructionSuggestions diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py index 608a73a5bc0057..dbfa99a892d003 100644 --- a/api/extensions/ext_application_services.py +++ b/api/extensions/ext_application_services.py @@ -31,6 +31,7 @@ build_data_source_services, ) from extensions.application_services.datasets import build_dataset_dependencies +from extensions.application_services.human_input import build_human_input_debug_service from extensions.application_services.installed_app import InstalledAppServices, build_installed_app_services from extensions.application_services.knowledge import ( KnowledgeServices, @@ -39,9 +40,15 @@ ) from extensions.application_services.rbac import RBACServices, build_rbac_services from extensions.application_services.resource_access_token import build_resource_access_token_service +from extensions.application_services.snippets import build_snippet_generation_service from extensions.application_services.tools import ToolServices, build_tool_services from extensions.application_services.trial_app import TrialAppServices, build_trial_app_services -from extensions.application_services.workflow import build_workflow_execution_dependencies, build_workflow_suggestions +from extensions.application_services.workflow import ( + build_console_workflow_service, + build_workflow_drafts, + build_workflow_execution_dependencies, + build_workflow_suggestions, +) from extensions.application_services.workflow_variables import ( build_console_workflow_variables, build_workflow_variable_service, @@ -56,10 +63,12 @@ from libs.helper import RateLimiter from libs.passport import PassportService from models.model import EndUser +from models.tool_runtime_contracts import WorkflowToolStore from models.utils.file_input_compat import restore_input_files from repositories.account.repository import SQLAlchemyAccountRepository from repositories.account_activation_repository import SQLAlchemyAccountActivationRepository from repositories.account_integration_repository import SQLAlchemyAccountIntegrationRepository +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from repositories.annotation_import_job_repository import RedisAnnotationImportJobRepository from repositories.annotation_reply_job_repository import RedisAnnotationReplyJobRepository from repositories.annotation_repository import AnnotationRepository @@ -98,7 +107,7 @@ from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository from repositories.web_passport_repository import WebPassportRepository from repositories.webapp_access_query_repository import WebAppAccessQueryRepository -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository from repositories.workspace.workspace_repository import WorkspaceRepository from services.account.adapters import ( @@ -115,6 +124,7 @@ from services.account.service import AccountSetupProvisioner from services.account_password_hasher import DefaultAccountPasswordHasher from services.agent.roster_package_exporter import RosterAgentPackageExporter +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.agent_tracing_config_service import AgentTracingConfigService from services.annotation_command_service import AnnotationCommandService from services.annotation_import_service import AnnotationImportLimits, AnnotationImportQuota, AnnotationImportService @@ -162,6 +172,7 @@ from services.file_grant_gateways import FileGrantFileGateway, FileGrantRemoteFileGateway, FileGrantTokenGateway from services.file_grant_service import FileGrantService from services.file_service import FileService +from services.human_input.debug_service import HumanInputDebugService from services.human_input_file_upload_service import HumanInputFileUploadService from services.init_validation_service import InitValidationService from services.inner_mail_service import InnerMailService @@ -220,6 +231,7 @@ from services.schema_definition_service import SchemaDefinitionService from services.setup_adapters import RedisSetupLock from services.setup_service import SetupService +from services.snippet_generate_service import SnippetGenerateService from services.step_by_step_tour_service import StepByStepTourService from services.system_feature_service import SystemFeatureService from services.tag_application_service import TagApplicationService @@ -242,10 +254,12 @@ from services.web_passport_service import WebPassportService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService, WebAppAccessUnavailableError +from services.workflow.app_log_query_service import WorkflowAppLogQueryService +from services.workflow.console_service import ConsoleWorkflowService from services.workflow.console_variable_service import ConsoleWorkflowVariableService +from services.workflow.draft_service import WorkflowDraftService from services.workflow.generation.suggestions import WorkflowInstructionSuggestions from services.workflow.variable_service import WorkflowVariableService -from services.workflow_app_log_query_service import WorkflowAppLogQueryService from services.workflow_run_service import WorkflowRunService from services.workflow_statistic_query_service import WorkflowStatisticQueryService from tasks.annotation.add_annotation_to_index_task import add_annotation_to_index_task @@ -304,10 +318,15 @@ class AppScopedEndUserServices: @dataclass(frozen=True, slots=True) class ApplicationServices: + human_input_debug: HumanInputDebugService + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + workflow_drafts: WorkflowDraftService[WorkflowAgentBindingStore] + snippet_generation: SnippetGenerateService tools: ToolServices workflow_runtime: AppGenerationRuntime workflow_variables: WorkflowVariableService console_workflow_variables: ConsoleWorkflowVariableService + workflow_agent_bindings: WorkflowAgentBindingResolver workflow_suggestions: WorkflowInstructionSuggestions annotation_commands: AnnotationCommandService annotation_imports: AnnotationImportService @@ -525,6 +544,8 @@ def build_application_services( providers=datasource_credentials.providers, ) oauth_server = _build_oauth_server_service(database_client=database_client, redis=redis) + tools = build_tool_services(database_client) + workflow_variables = build_workflow_variable_service(database_client=database_client) console_apps = ConsoleAppRepository(session_factory=database_client) rbac = build_rbac_services( workspaces=workspace_repository, @@ -532,6 +553,12 @@ def build_application_services( apps=console_apps, datasets=dataset_dependencies.datasets, ) + console_workflows = build_console_workflow_service( + database_client=database_client, + redis=redis, + variables=workflow_variables, + rbac_members=rbac.members, + ) tracing_configs = AppTracingConfigService( configs=SQLAlchemyAppTracingConfigRepository(session_factory=database_client), provider=OpsTraceManagerGateway(), @@ -540,12 +567,16 @@ def build_application_services( database_client=database_client, oauth=oauth_server, recommended_packages=recommended_app_packages, + tool_providers=tools.tool_providers, + workflow_tools=cast(WorkflowToolStore, tools.workflow_queries), + workflow_publisher=console_workflows, repository=console_apps, rbac_members=rbac.members, tracing_configs=tracing_configs, ) tags = TagApplicationService(tags=TagRepository(session_factory=database_client)) knowledge = build_knowledge_services( + variables=workflow_variables, database_client=database_client, dataset_access=dataset_dependencies.access, datasets=dataset_dependencies.datasets, @@ -599,7 +630,6 @@ def build_application_services( registration=account_services.lifecycle, invitation_tokens=invitation_tokens, ) - workflow_variables = build_workflow_variable_service(database_client=database_client) conversations = ConversationRepository(session_factory=database_client) conversation_names = ConversationNameService(conversations=conversations, generate_name=generate_conversation_name) conversation_commands = ConversationCommandService( @@ -618,13 +648,20 @@ def build_application_services( ) return ApplicationServices( - tools=build_tool_services(database_client), - workflow_runtime=build_workflow_execution_dependencies(database_client), + tools=tools, + human_input_debug=build_human_input_debug_service(database_client=database_client), + console_workflows=console_workflows, + workflow_drafts=build_workflow_drafts(database_client), workflow_variables=workflow_variables, console_workflow_variables=build_console_workflow_variables( database_client=database_client, variables=workflow_variables ), + workflow_runtime=build_workflow_execution_dependencies(database_client), workflow_suggestions=build_workflow_suggestions(database_client), + workflow_agent_bindings=WorkflowAgentBindingResolver(database_client), + snippet_generation=build_snippet_generation_service( + database_client=database_client, variables=workflow_variables + ), annotation_commands=AnnotationCommandService( annotations=annotations, add_index=add_annotation_to_index_task.delay, @@ -658,13 +695,15 @@ def build_application_services( accounts=account_services, apps=apps, credential_queries=CredentialQueryRepository(session_factory=database_client), - agent_apps=build_agent_app_services(database_client=database_client), + agent_apps=build_agent_app_services(database_client=database_client, variables=workflow_variables), advanced_prompt_templates=AdvancedPromptTemplateService(), app_definitions=app_definitions, app_mcp_servers=AppMCPServerService( servers=AppMCPServerRepository(session_factory=database_client), ), - app_preview_details=AppPreviewDetailsRuntime(details=app_preview_repository), + app_preview_details=AppPreviewDetailsRuntime( + details=app_preview_repository, tool_providers=tools.tool_providers, workflow_queries=tools.workflow_queries + ), app_previews=AppPreviewQueryService( apps=app_preview_repository, is_previewable=recommended_app_queries.is_previewable, @@ -729,6 +768,7 @@ def build_application_services( private_app_access_enabled=deployment_edition == DeploymentEdition.ENTERPRISE, ), installed_apps=build_installed_app_services( + variables=workflow_variables, conversation_queries=conversation_queries, conversation_names=conversation_names, conversation_commands=conversation_commands, @@ -839,7 +879,9 @@ def build_application_services( ), app_tasks=AppTaskControlService(redis_client=redis), app_audio=AppAudioRuntime(session_factory=database_client), - trial_apps=build_trial_app_services(database_client=database_client, trial_apps=trial_apps), + trial_apps=build_trial_app_services( + database_client=database_client, trial_apps=trial_apps, variables=workflow_variables + ), workflow_run_archives=WorkflowRunArchiveService( bundles=WorkflowRunArchiveBundleQueryRepository(session_factory=database_client), tasks=WorkflowRunArchiveDownloadTaskCache(redis=redis), @@ -850,11 +892,12 @@ def build_application_services( workflow_runs=WorkflowRunService( workflow_runs=workflow_run_repository, node_executions=workflow_node_execution_repository, + tool_providers=tools.tool_providers, session_factory=database_client, ), workspaces=workspace_services, workflow_app_logs=WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=database_client), + logs=WorkflowAppLogRepository(session_factory=database_client), ), workflow_comments=WorkflowCommentService( comments=WorkflowCommentRepository(session_factory=database_client), diff --git a/api/extensions/workflow_warm_shutdown.py b/api/extensions/workflow_warm_shutdown.py index e6579ccdb5d898..c0722f352e81f7 100644 --- a/api/extensions/workflow_warm_shutdown.py +++ b/api/extensions/workflow_warm_shutdown.py @@ -6,7 +6,7 @@ from celery.signals import worker_shutdown, worker_shutting_down -from core.app.apps.workflow.active_workflow_tasks import ( +from services.workflow.execution.adapters.workflow.active_workflow_tasks import ( get_active_workflow_task_count, reset_active_workflow_tasks, ) diff --git a/api/fields/agent_fields.py b/api/fields/agent_fields.py index 1988ccb42d0acd..d01456142846be 100644 --- a/api/fields/agent_fields.py +++ b/api/fields/agent_fields.py @@ -3,6 +3,7 @@ from pydantic import Field, field_validator +from enums.agent import WorkflowAgentBindingType from fields.base import ResponseModel from libs.helper import to_timestamp from models.agent import ( @@ -13,7 +14,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, ) from models.agent_config_entities import ( AgentCliToolConfig, diff --git a/api/fields/workflow_run_fields.py b/api/fields/workflow_run_fields.py index c7878c165bce02..e8a0c87d15255d 100644 --- a/api/fields/workflow_run_fields.py +++ b/api/fields/workflow_run_fields.py @@ -17,6 +17,9 @@ from fields.end_user_fields import SimpleEndUser from fields.member_fields import SimpleAccount from libs.helper import to_timestamp +from models.workflow import WorkflowNodeExecutionModel +from services.tools.provider_queries import ToolProviderIcons +from services.workflow.node_execution_extras import node_execution_extras class WorkflowRunForLogResponse(ResponseModel): @@ -212,6 +215,16 @@ def workflow_run_pagination_response_source(pagination: Any, *, session: Session class WorkflowNodeExecutionResponseSource(SessionResponseSource[Any]): """Expose session-backed node-execution accessors during response validation.""" + def __init__(self, source: Any, *, session: Session, tool_providers: ToolProviderIcons): + super().__init__(source, session=session) + self._tool_providers = tool_providers + + @property + def extras(self) -> Any: + if isinstance(self._source, WorkflowNodeExecutionModel): + return node_execution_extras(self._source, tool_providers=self._tool_providers) + return self._source.extras + @property def created_by_account(self) -> Any: return self._source.created_by_account(self._session) @@ -221,8 +234,10 @@ def created_by_end_user(self) -> Any: return self._source.created_by_end_user(self._session) -def node_execution_response_source(node_execution: Any, *, session: Session) -> WorkflowNodeExecutionResponseSource: - return WorkflowNodeExecutionResponseSource(node_execution, session=session) +def node_execution_response_source( + node_execution: Any, *, session: Session, tool_providers: ToolProviderIcons +) -> WorkflowNodeExecutionResponseSource: + return WorkflowNodeExecutionResponseSource(node_execution, session=session, tool_providers=tool_providers) class WorkflowRunNodeExecutionListResponse(ResponseModel): diff --git a/api/libs/helper.py b/api/libs/helper.py index a1bf6b31d2a38e..023f7393c3af94 100644 --- a/api/libs/helper.py +++ b/api/libs/helper.py @@ -10,6 +10,7 @@ from collections.abc import Callable, Generator, Iterable, Mapping from contextlib import contextmanager from datetime import datetime +from functools import partial from hashlib import sha256 from typing import TYPE_CHECKING, Annotated, Any, Protocol, cast, overload, override from uuid import UUID @@ -27,6 +28,7 @@ from extensions.redis_names import serialize_redis_name from graphon.file import helpers as file_helpers from graphon.model_runtime.utils.encoders import jsonable_encoder +from libs.stream import close_stream if TYPE_CHECKING: from models import Account @@ -419,11 +421,15 @@ def compact_generate_response( def generate() -> Generator[str]: yield from stream_response - return Response( + result = Response( _stream_with_request_context(generate()), status=200, mimetype="text/event-stream", ) + # Closing an unconsumed wrapper does not enter its yield-from scope. + # Explicitly forward response closure to streams that own resources. + result.call_on_close(partial(close_stream, stream_response)) + return result def length_prefixed_response( diff --git a/api/models/model.py b/api/models/model.py index 88b1909b03ec4f..6ad99293c96a26 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -52,7 +52,6 @@ ProviderQuotaType, TagType, ) -from .provider_ids import GenericProviderID from .types import EnumText, LongText, StringUUID if TYPE_CHECKING: @@ -171,12 +170,6 @@ class FileUploadConfig(TypedDict): image: ImageUploadConfig -class DeletedToolInfo(TypedDict): - type: str - tool_name: str - provider_id: str - - class ExternalDataToolConfig(TypedDict): enabled: bool variable: str @@ -580,118 +573,6 @@ def mode_compatible_with_agent_with_session(self, *, session: Session) -> str: return str(self.mode) - def deleted_tools_with_session(self, *, session: Session) -> list[DeletedToolInfo]: - from core.plugin.plugin_service import PluginService - from core.tools.tool_manager import ToolManager, ToolProviderType - - # get agent mode tools - app_model_config = self.app_model_config_with_session(session=session) - if not app_model_config: - return [] - - if not app_model_config.agent_mode: - return [] - - agent_mode = app_model_config.agent_mode_dict - tools = agent_mode.get("tools", []) - - api_provider_ids: list[str] = [] - - builtin_provider_ids: list[GenericProviderID] = [] - - for tool in tools: - keys = list(tool.keys()) - if len(keys) >= 4: - provider_type = tool.get("provider_type", "") - provider_id = tool.get("provider_id", "") - if provider_type == ToolProviderType.API: - try: - uuid.UUID(provider_id) - except Exception: - continue - api_provider_ids.append(provider_id) - if provider_type == ToolProviderType.BUILT_IN: - try: - # check if it's hardcoded - try: - ToolManager.get_hardcoded_provider(provider_id) - is_hardcoded = True - except Exception: - is_hardcoded = False - - provider_id = GenericProviderID(provider_id, is_hardcoded) - except Exception: - continue - - builtin_provider_ids.append(provider_id) - - if not api_provider_ids and not builtin_provider_ids: - return [] - - if api_provider_ids: - existing_api_providers = [ - str(api_provider.id) - for api_provider in session.execute( - text("SELECT id FROM tool_api_providers WHERE id IN :provider_ids"), - {"provider_ids": tuple(api_provider_ids)}, - ).fetchall() - ] - else: - existing_api_providers = [] - - if builtin_provider_ids: - # get the non-hardcoded builtin providers - non_hardcoded_builtin_providers = [ - provider_id for provider_id in builtin_provider_ids if not provider_id.is_hardcoded - ] - if non_hardcoded_builtin_providers: - existence = list(PluginService.check_tools_existence(self.tenant_id, non_hardcoded_builtin_providers)) - else: - existence = [] - # add the hardcoded builtin providers - existence.extend([True] * (len(builtin_provider_ids) - len(non_hardcoded_builtin_providers))) - builtin_provider_ids = non_hardcoded_builtin_providers + [ - provider_id for provider_id in builtin_provider_ids if provider_id.is_hardcoded - ] - else: - existence = [] - - existing_builtin_providers = { - provider_id.provider_name: existence[i] for i, provider_id in enumerate(builtin_provider_ids) - } - - deleted_tools: list[DeletedToolInfo] = [] - - for tool in tools: - keys = list(tool.keys()) - if len(keys) >= 4: - provider_type = tool.get("provider_type", "") - provider_id = tool.get("provider_id", "") - - if provider_type == ToolProviderType.API: - if provider_id not in existing_api_providers: - deleted_tools.append( - { - "type": ToolProviderType.API, - "tool_name": tool["tool_name"], - "provider_id": provider_id, - } - ) - - if provider_type == ToolProviderType.BUILT_IN: - generic_provider_id = GenericProviderID(provider_id) - - if not existing_builtin_providers[generic_provider_id.provider_name]: - deleted_tools.append( - { - "type": ToolProviderType.BUILT_IN, - "tool_name": tool["tool_name"], - "provider_id": provider_id, # use the original one - } - ) - - return deleted_tools - def tags_with_session(self, *, session: Session) -> Sequence[Tag]: tags = session.scalars( select(Tag) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index c76ad5f0d8295c..33f4fbfc56d37a 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -2049,8 +2049,6 @@ Get advanced chat workflow run list | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for advanced chat workflow #### Parameters @@ -2071,12 +2069,9 @@ Get human input form preview for advanced chat workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for advanced chat workflow #### Parameters @@ -2097,7 +2092,6 @@ Submit human input form preview for advanced chat workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/message-template/test @@ -2123,8 +2117,6 @@ Submit human input form preview for advanced chat workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** - Run draft workflow iteration node for advanced chat #### Parameters @@ -2146,12 +2138,10 @@ Run draft workflow iteration node for advanced chat | ---- | ----------- | ------ | | 200 | Iteration node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** - Run draft workflow loop node for advanced chat #### Parameters @@ -2173,12 +2163,10 @@ Run draft workflow loop node for advanced chat | ---- | ----------- | ------ | | 200 | Loop node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/advanced-chat/workflows/draft/run -**Run draft workflow** - Run draft workflow for advanced chat application #### Parameters @@ -2200,7 +2188,6 @@ Run draft workflow for advanced chat application | 200 | Workflow run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Invalid request parameters | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -3126,11 +3113,7 @@ Get conversation variables for an application | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/convert-to-workflow -**Convert basic mode of chatbot app to workflow mode** - Convert application to workflow mode -Convert expert mode of chatbot app to workflow mode -Convert Completion App to Workflow App #### Parameters @@ -3151,7 +3134,6 @@ Convert Completion App to Workflow App | 200 | Application converted to workflow successfully | **application/json**: [NewAppResponse](#newappresponse)
| | 400 | Application cannot be converted | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/copy @@ -4085,8 +4067,6 @@ Get workflow application execution logs | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflow-runs/tasks/{task_id}/stop -**Stop workflow task** - Stop running workflow task #### Parameters @@ -4102,7 +4082,7 @@ Stop running workflow task | ---- | ----------- | ------ | | 200 | Task stopped successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Task not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Task not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflow-runs/{run_id} @@ -4523,8 +4503,6 @@ Get workflow daily token cost statistics | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows -**Get published workflows** - Get all published workflows for an application #### Parameters @@ -4542,12 +4520,9 @@ Get all published workflows for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Published workflows retrieved successfully | **application/json**: [WorkflowPaginationResponse](#workflowpaginationresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs -**Get default block config** - Get default block configurations for workflow #### Parameters @@ -4561,12 +4536,9 @@ Get default block configurations for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Default block configurations retrieved successfully | **application/json**: [DefaultBlockConfigsResponse](#defaultblockconfigsresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/default-workflow-block-configs/{block_type} -**Get default block config** - Get default block configuration by type #### Parameters @@ -4582,12 +4554,10 @@ Get default block configuration by type | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Default block configuration retrieved successfully | **application/json**: [DefaultBlockConfigResponse](#defaultblockconfigresponse)
| -| 404 | Block type not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Block type not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/draft -**Get draft workflow** - Get draft workflow for an application #### Parameters @@ -4601,12 +4571,10 @@ Get draft workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Draft workflow retrieved successfully | **application/json**: [WorkflowResponse](#workflowresponse)
| -| 404 | Draft workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[DraftWorkflowNotExistError](#draftworkflownotexisterror)
[ErrorResponse](#errorresponse)
| +| 404 | Draft workflow not found | **application/json**: [DraftWorkflowNotExistError](#draftworkflownotexisterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft -**Sync draft workflow** - Sync draft workflow configuration #### Parameters @@ -4628,7 +4596,6 @@ Sync draft workflow configuration | 200 | Draft workflow synced successfully | **application/json**: [SyncDraftWorkflowResponse](#syncdraftworkflowresponse)
| | 400 | Invalid workflow configuration | **application/json**: [ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 409 | Conflict | **application/json**: [DraftWorkflowNotSyncError](#draftworkflownotsyncerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -4732,12 +4699,9 @@ Update draft workflow features | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Workflow features updated successfully | **application/json**: [SimpleResultResponse](#simpleresultresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test -**Test human input delivery** - Test human input delivery for workflow #### Parameters @@ -4758,12 +4722,9 @@ Test human input delivery for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input delivery test result | **application/json**: [EmptyObjectResponse](#emptyobjectresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview -**Preview human input form content and placeholders** - Get human input form preview for workflow #### Parameters @@ -4784,12 +4745,9 @@ Get human input form preview for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form preview | **application/json**: [HumanInputFormPreviewResponse](#humaninputformpreviewresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run -**Submit human input form preview** - Submit human input form preview for workflow #### Parameters @@ -4810,7 +4768,6 @@ Submit human input form preview for workflow | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Human input form submission result | **application/json**: [HumanInputFormSubmitResponse](#humaninputformsubmitresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/message-template/test @@ -4836,7 +4793,7 @@ Submit human input form preview for workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run -**Run draft workflow iteration node** +Run draft workflow iteration node #### Parameters @@ -4857,11 +4814,11 @@ Submit human input form preview for workflow | ---- | ----------- | ------ | | 200 | Workflow iteration node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run -**Run draft workflow loop node** +Run draft workflow loop node #### Parameters @@ -4882,7 +4839,7 @@ Submit human input form preview for workflow | ---- | ----------- | ------ | | 200 | Workflow loop node run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer @@ -5051,11 +5008,11 @@ Get last run result for draft workflow node | ---- | ----------- | ------ | | 200 | Node last run retrieved successfully | **application/json**: [WorkflowRunNodeExecutionResponse](#workflowrunnodeexecutionresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node last run not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node last run not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/run -**Run draft workflow node** +Run draft workflow node #### Parameters @@ -5076,11 +5033,11 @@ Get last run result for draft workflow node | ---- | ----------- | ------ | | 200 | Node run started successfully | **application/json**: [WorkflowRunNodeExecutionResponse](#workflowrunnodeexecutionresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Node not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Node not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run -**Poll for trigger events and execute single node when event arrives** +Poll for trigger events and execute single node when event arrives #### Parameters @@ -5096,7 +5053,6 @@ Get last run result for draft workflow node | 200 | Trigger event received and node executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5135,7 +5091,7 @@ Get variables for a specific node | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/run -**Run draft workflow** +Run draft workflow #### Parameters @@ -5155,7 +5111,6 @@ Get variables for a specific node | ---- | ----------- | ------ | | 200 | Draft workflow run started successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5251,7 +5206,7 @@ Get system variables for workflow | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/trigger/run -**Poll for trigger events and execute full workflow when event arrives** +Poll for trigger events and execute full workflow when event arrives #### Parameters @@ -5272,13 +5227,12 @@ Get system variables for workflow | 200 | Trigger event received and workflow executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/draft/trigger/run-all -**Full workflow debug when the start node is a trigger** +Full workflow debug when the start node is a trigger #### Parameters @@ -5299,7 +5253,6 @@ Get system variables for workflow | 200 | Workflow executed successfully | **application/json**: [GeneratedAppResponse](#generatedappresponse)
| | 400 | Trigger run failed | **application/json**: [WorkflowTriggerRunErrorResponse](#workflowtriggerrunerrorresponse)
[ErrorResponse](#errorresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | 429 | Too Many Requests | **application/json**: [RateLimitError](#ratelimiterror)
[ErrorResponse](#errorresponse)
| | 500 | Internal server error | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -5421,8 +5374,6 @@ Reset a workflow variable to its default value | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/publish -**Get published workflow** - Get published workflow for an application #### Parameters @@ -5436,12 +5387,9 @@ Get published workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Published workflow retrieved successfully, or null if not found | **application/json**: [WorkflowResponse](#workflowresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/publish -**Publish workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -5459,7 +5407,6 @@ Get published workflow for an application | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Workflow published successfully | **application/json**: [WorkflowPublishResponse](#workflowpublishresponse)
| -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs @@ -5556,8 +5503,6 @@ Full value for one declared output of a published run. | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /apps/{app_id}/workflows/{workflow_id} -**Delete workflow** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -5570,12 +5515,9 @@ Full value for one declared output of a published run. | Code | Description | Schema | | ---- | ----------- | ------ | | 204 | Workflow deleted successfully | | -| 404 | Not Found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [PATCH] /apps/{app_id}/workflows/{workflow_id} -**Update workflow attributes** - Update workflow by ID #### Parameters @@ -5597,7 +5539,7 @@ Update workflow by ID | ---- | ----------- | ------ | | 200 | Workflow updated successfully | **application/json**: [WorkflowResponse](#workflowresponse)
| | 403 | Permission denied | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Workflow not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /apps/{app_id}/workflows/{workflow_id}/restore @@ -5616,7 +5558,7 @@ Restore a published workflow version into the draft workflow | ---- | ----------- | ------ | | 200 | Workflow restored successfully | **application/json**: [WorkflowRestoreResponse](#workflowrestoreresponse)
| | 400 | Source workflow must be published | **application/json**: [ErrorResponse](#errorresponse)
| -| 404 | Workflow not found | **application/json**: [AppNotFoundError](#appnotfounderror)
[ErrorResponse](#errorresponse)
| +| 404 | Workflow not found | **application/json**: [ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [GET] /apps/{resource_id}/api-keys @@ -11869,7 +11811,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowSyncResponse](#ragpipelineworkflowsyncresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | 409 | Conflict | **application/json**: [DraftWorkflowNotSyncError](#draftworkflownotsyncerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| @@ -12247,7 +12188,7 @@ Update account-level Step-by-step Tour state | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/publish -**Publish workflow** +**Publish a prepared Pipeline and its dataset settings atomically** #### Parameters @@ -12260,7 +12201,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowPublishResponse](#ragpipelineworkflowpublishresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /rag/pipelines/{pipeline_id}/workflows/published/datasource/nodes/{node_id}/preview @@ -12372,8 +12312,6 @@ Update account-level Step-by-step Tour state | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /rag/pipelines/{pipeline_id}/workflows/{workflow_id} -**Delete a published workflow version that is not currently active on the pipeline** - #### Parameters | Name | Located in | Description | Required | Schema | @@ -12386,7 +12324,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 204 | Workflow deleted successfully | | -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [PATCH] /rag/pipelines/{pipeline_id}/workflows/{workflow_id} @@ -12428,7 +12365,6 @@ Update account-level Step-by-step Tour state | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Success | **application/json**: [RagPipelineWorkflowSyncResponse](#ragpipelineworkflowsyncresponse)
| -| 404 | Not Found | **application/json**: [PipelineNotFoundError](#pipelinenotfounderror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [POST] /refresh-token @@ -13292,7 +13228,9 @@ Reset a draft workflow variable to its default value (snippet scope) | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| ### [DELETE] /snippets/{snippet_id}/workflows/{workflow_id} -**Delete a published snippet workflow version** +**Delete a version and retire its unowned Agents through the shared use case** + +Delete a published snippet workflow version #### Parameters diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index 0962c702afd98b..e717aae3a90d03 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -2746,7 +2746,7 @@ Execute the full knowledge pipeline for a knowledge base. Published runs are que | 200 | Pipeline execution result. Published runs return a JSON object containing `batch`, `dataset`, and `documents`. Draft runs return `text/event-stream` for streaming mode or a workflow result JSON object for blocking mode. | **application/json**: [PipelineRunJsonResponse](#pipelinerunjsonresponse)
**text/event-stream**: string
| | 400 | Bad request - invalid payload or pipeline is not configured | **application/json**: [ErrorResponse](#errorresponse)
| | 401 | Unauthorized - invalid API token | **application/json**: [ErrorResponse](#errorresponse)
| -| 403 | `forbidden` : Forbidden. | **application/json**: [ErrorResponse](#errorresponse)
| +| 403 | `forbidden` : AccessDeniedError. | **application/json**: [ErrorResponse](#errorresponse)
| | 404 | `not_found` : Dataset not found. | **application/json**: [ErrorResponse](#errorresponse)
| | 500 | `pipeline_run_error` : Pipeline execution failed. | **application/json**: [PipelineRunError](#pipelinerunerror)
[ErrorResponse](#errorresponse)
| | default | Error | **application/json**: [ErrorResponse](#errorresponse)
| diff --git a/api/repositories/app/dsl_import_repository.py b/api/repositories/app/dsl_import_repository.py index 646dbaff170ab4..9439f254b62f6d 100644 --- a/api/repositories/app/dsl_import_repository.py +++ b/api/repositories/app/dsl_import_repository.py @@ -9,21 +9,23 @@ from graphon.variables import VariableBase from machinery.context import RequestContext from repositories.app.console_repository import require_console_app +from repositories.credentials.query_repository import CredentialQueryRepository +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.workflow.dsl_import import DslImportStore, DslImportTarget, stored_secret_ids from services.workflow.graph_check import ResourceCheck, credential_check from services.workflow.graph_diff import draft_token -from services.workflow_service import WorkflowService class DslImportRepository(DslImportStore): def __init__(self, *, session_factory: sessionmaker[Session]) -> None: self._session_factory = session_factory + self._credential_queries = CredentialQueryRepository(session_factory=session_factory) @override def load_target(self, context: RequestContext, app_id: str) -> DslImportTarget: with self._session_factory() as session: app = require_console_app(session, context, app_id) - draft = WorkflowService.get_draft_workflow(app, session=session) + draft = WorkflowDefinitionStore.get_draft_workflow(app, session=session) return DslImportTarget( mode=app.mode, draft_token=draft_token(draft) if draft else None, @@ -33,5 +35,4 @@ def load_target(self, context: RequestContext, app_id: str) -> DslImportTarget: @override @contextmanager def credential_check(self, workspace_id: str, environment: Mapping[str, VariableBase]) -> Generator[ResourceCheck]: - with self._session_factory() as session: - yield credential_check(workspace_id, environment, session) + yield credential_check(workspace_id, environment, self._credential_queries) diff --git a/api/repositories/app/generation_repository.py b/api/repositories/app/generation_repository.py index cf0659260b3c65..fcdff3af2dc995 100644 --- a/api/repositories/app/generation_repository.py +++ b/api/repositories/app/generation_repository.py @@ -392,7 +392,9 @@ def save_message(self, identity: MessageIdentity, update: MessageUpdate) -> Mess for fields in update.files ) session.flush() - session.refresh(message) + # Every field consumed by post-commit message subscribers is assigned + # above. A full refresh would expire unloaded relationships before the + # row is detached, leaving subscribers with an unusable detached model. session.expunge(message) return message diff --git a/api/repositories/app_definition_query_repository.py b/api/repositories/app_definition_query_repository.py index 703b28bf979978..81f7c534c80707 100644 --- a/api/repositories/app_definition_query_repository.py +++ b/api/repositories/app_definition_query_repository.py @@ -9,13 +9,13 @@ from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features from core.app.apps.agent_app.app_variable_projection import agent_app_variables_to_user_input_form -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus from models.agent import AgentConfigSnapshot from models.agent_config_entities import AgentSoulConfig from models.model import App, AppMode, AppModelConfig, Site, load_annotation_reply_config from models.tools import ApiToolProvider from models.workflow import Workflow +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionQuery, AppDefinitionSummary, diff --git a/api/repositories/sqlalchemy_execution_extra_content_repository.py b/api/repositories/sqlalchemy_execution_extra_content_repository.py index bb36d7d336ffb1..3cccd18a3fe715 100644 --- a/api/repositories/sqlalchemy_execution_extra_content_repository.py +++ b/api/repositories/sqlalchemy_execution_extra_content_repository.py @@ -17,7 +17,6 @@ HumanInputFormDefinition, HumanInputFormSubmissionData, ) -from core.workflow.nodes.human_input.callback import DifyHITLCallback from enums.human_input import HumanInputFormStatus, RecipientType from models.execution_extra_content import ( ExecutionExtraContent as ExecutionExtraContentModel, @@ -32,6 +31,7 @@ HumanInputFormReference, StoredExecutionExtraContent, ) +from services.workflow.execution.adapters.human_input import DifyHITLCallback logger = logging.getLogger(__name__) diff --git a/api/repositories/workflow/definition_repository.py b/api/repositories/workflow/definition_repository.py index 85c275a7205a5c..55e47b1c37ef95 100644 --- a/api/repositories/workflow/definition_repository.py +++ b/api/repositories/workflow/definition_repository.py @@ -27,9 +27,7 @@ from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowKind, WorkflowType from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository, workflow_binding_scope -from repositories.app.console_repository import console_app_actor, require_console_app from repositories.trigger.workflow_repository import WorkflowTriggerRepository -from repositories.workflow.draft_variable_repository import delete_workflow_variables from repositories.workflow.version_repository import allocate_version_number from services.errors.app import ( IsDraftWorkflowError, @@ -118,6 +116,8 @@ def get_execution_workflow(app_model: App, *, workflow_id: str | None, draft: bo @staticmethod def lock_app(session: Session, context: RequestContext, app_id: str) -> App: + from repositories.app.console_repository import require_console_app + app = require_console_app(session, context, app_id) return session.scalars( select(App) @@ -130,6 +130,8 @@ def lock_app(session: Session, context: RequestContext, app_id: str) -> App: def owner( session: Session, context: RequestContext, owner: WorkflowOwner, *, lock: bool = False ) -> App | Pipeline | CustomizedSnippet: + from repositories.app.console_repository import require_console_app + if owner.kind == "app": return ( WorkflowDefinitionStore.lock_app(session, context, owner.id) @@ -535,6 +537,8 @@ def sync_draft_workflow( app_model.updated_at = naive_utc_now() if clear_debug_variables: # DSL replacement and debug-variable invalidation share one atomic write. + from repositories.workflow.draft_variable_repository import delete_workflow_variables + delete_workflow_variables(session, app_id=app_model.id, user_id=None, node_id=None) session.flush() if isinstance(app_model, Pipeline) and current is None: @@ -712,6 +716,7 @@ def workflow_from_snapshot(snapshot: WorkflowSnapshot) -> Workflow: @dataclass class _PublicationTransaction: session: Session + session_maker: sessionmaker[Session] app: App actor: Account draft: WorkflowSnapshot @@ -744,6 +749,15 @@ def sync_triggers(self, triggers: Sequence[Mapping[str, Any]]) -> None: def sync_datasets(self, dataset_ids: set[str]) -> None: WorkflowDefinitionStore.sync_dataset_relationships(self.session, self.app, dataset_ids) + def sync_knowledge_spaces(self, workflow: WorkflowSnapshot) -> None: + from services.agent.knowledge_spaces import sync_workflow_app_knowledge_bindings + + sync_workflow_app_knowledge_bindings( + session=self.session, + session_maker=self.session_maker, + workflow=workflow_from_snapshot(workflow), + ) + def activate(self, workflow: WorkflowSnapshot) -> None: WorkflowDefinitionStore.activate(self.session, self.app, workflow.id, self.actor.id) @@ -767,6 +781,8 @@ def debug_context( snapshot: WorkflowSnapshot | None, ) -> tuple[App, Account, WorkflowSnapshot]: """Load execution inputs in a short read, keeping persisted secrets encrypted.""" + from repositories.app.console_repository import console_app_actor, require_console_app + if snapshot is not None and (snapshot.tenant_id, snapshot.app_id) != (context.active_workspace_id, app_id): raise WorkflowNotFoundError("Workflow not found") with self._sessions() as session: @@ -792,6 +808,8 @@ def get_published_workflow(self, pipeline: Pipeline) -> Workflow | None: def validation_state( self, context: RequestContext, app_id: str, workflow_id: str | None = None ) -> tuple[App, WorkflowSnapshot | None]: + from repositories.app.console_repository import require_console_app + with nullcontext(self._import_session) if self._import_session is not None else self._sessions() as session: app = require_console_app(session, context, app_id) workflow = WorkflowDefinitionStore.get_draft_workflow(app, workflow_id, session=session) @@ -803,6 +821,8 @@ def binding_reader(self) -> Generator[WorkflowAgentBindingRepository]: yield WorkflowAgentBindingRepository(session) def app(self, context: RequestContext, app_id: str) -> App: + from repositories.app.console_repository import require_console_app + with self._sessions() as session: return require_console_app(session, context, app_id) @@ -810,6 +830,8 @@ def app(self, context: RequestContext, app_id: str) -> App: def draft( self, context: RequestContext, app_id: str ) -> Generator[DraftWorkflowRead[WorkflowAgentBindingRepository] | None]: + from repositories.app.console_repository import require_console_app + with self._sessions() as session: app = require_console_app(session, context, app_id) workflow = WorkflowDefinitionStore.get_draft_workflow(app, session=session) @@ -827,6 +849,8 @@ def draft( def publication( self, context: RequestContext, app_id: str, draft: WorkflowSnapshot ) -> Generator[WorkflowPublicationTransaction[WorkflowAgentBindingRepository]]: + from repositories.app.console_repository import console_app_actor + with self._sessions.begin() as session: app = WorkflowDefinitionStore.lock_app(session, context, app_id) current = WorkflowDefinitionStore.get_draft_workflow(app, session=session, for_update=True) @@ -834,6 +858,7 @@ def publication( raise WorkflowHashNotEqualError() yield _PublicationTransaction( session, + self._sessions, app, console_app_actor(session, context), workflow_snapshot(current), @@ -841,6 +866,8 @@ def publication( ) def debug_node(self, context: RequestContext, app_id: str, node_id: str) -> WorkflowDebugNode: + from repositories.app.console_repository import require_console_app + with self._sessions() as session: app = require_console_app(session, context, app_id) workflow = WorkflowDefinitionStore.get_draft_workflow(app, session=session) @@ -856,6 +883,8 @@ def debug_node(self, context: RequestContext, app_id: str, node_id: str) -> Work ) def published(self, context: RequestContext, app_id: str) -> WorkflowRecord | None: + from repositories.app.console_repository import require_console_app + with self._sessions() as session: app = require_console_app(session, context, app_id) workflow = WorkflowDefinitionStore.get_published_workflow(app, session=session) @@ -864,6 +893,8 @@ def published(self, context: RequestContext, app_id: str) -> WorkflowRecord | No def versions( self, context: RequestContext, app_id: str, *, page: int, limit: int, user_id: str | None, named_only: bool ) -> tuple[list[WorkflowRecord], bool]: + from repositories.app.console_repository import require_console_app + with self._sessions() as session: app = require_console_app(session, context, app_id) workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( @@ -874,6 +905,8 @@ def versions( def update( self, context: RequestContext, app_id: str, workflow_id: str, changes: dict[str, str] ) -> WorkflowRecord | None: + from repositories.app.console_repository import require_console_app + with self._sessions.begin() as session: app = require_console_app(session, context, app_id) workflow = WorkflowDefinitionStore.update_workflow( @@ -903,6 +936,8 @@ def delete(self, context: RequestContext, owner: WorkflowOwner, workflow_id: str raise WorkflowNotFoundError(str(error)) from error def update_features(self, context: RequestContext, app_id: str, features: dict[str, Any]) -> None: + from repositories.app.console_repository import require_console_app + with self._sessions.begin() as session: app = require_console_app(session, context, app_id) WorkflowDefinitionStore.update_features( diff --git a/api/repositories/workflow_tool_source_repository.py b/api/repositories/workflow_tool_source_repository.py index cb6ee6a1993fde..50e165f8a025f1 100644 --- a/api/repositories/workflow_tool_source_repository.py +++ b/api/repositories/workflow_tool_source_repository.py @@ -1,49 +1,75 @@ -from collections.abc import Sequence +from collections.abc import Callable, Sequence from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker -from core.tools.workflow_as_tool.provider import WorkflowToolProviderController +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.tools.workflow_as_tool.repository import WorkflowToolSource -from core.tools.workflow_as_tool.tool import WorkflowTool from models.account import Account from models.model import App from models.tools import WorkflowToolProvider from models.workflow import Workflow +from services.tools.workflow.provider import WorkflowToolProviderController +from services.tools.workflow.tool import WorkflowTool +from services.workflow.execution.ports import WorkflowRuntime class SQLAlchemyWorkflowToolSourceRepository: - def __init__(self, session_maker: sessionmaker[Session]) -> None: + def __init__( + self, + session_maker: sessionmaker[Session], + *, + workflow_runtime: WorkflowRuntime | None, + draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None, + ) -> None: self._session_maker = session_maker + self._workflow_runtime = workflow_runtime + self._draft_variable_saver = draft_variable_saver def get_by_provider_ids(self, *, tenant_id: str, provider_ids: Sequence[str]) -> Sequence[WorkflowToolSource]: if not provider_ids: return () with self._session_maker() as session: - rows = session.execute( - select(Workflow, WorkflowToolProvider, Account.name) - .join(App, App.id == Workflow.app_id) - .join( - WorkflowToolProvider, - (WorkflowToolProvider.app_id == Workflow.app_id) - & (WorkflowToolProvider.version == Workflow.version), - ) - .outerjoin(Account, Account.id == WorkflowToolProvider.user_id) - .where( - App.tenant_id == tenant_id, - Workflow.tenant_id == tenant_id, - WorkflowToolProvider.tenant_id == tenant_id, - WorkflowToolProvider.id.in_(provider_ids), - Workflow.version != Workflow.VERSION_DRAFT, + rows = tuple( + session.execute( + select(Workflow, WorkflowToolProvider) + .join(App, App.id == Workflow.app_id) + .join( + WorkflowToolProvider, + (WorkflowToolProvider.app_id == Workflow.app_id) + & (WorkflowToolProvider.version == Workflow.version), + ) + .where( + App.tenant_id == tenant_id, + Workflow.tenant_id == tenant_id, + WorkflowToolProvider.tenant_id == tenant_id, + WorkflowToolProvider.id.in_(provider_ids), + Workflow.version != Workflow.VERSION_DRAFT, + ) ) ) - return tuple( - self._materialize( - workflow, - tool=WorkflowToolProviderController.build_tool(provider, workflow, author=author or ""), - ) - for workflow, provider, author in rows + return tuple( + self._materialize( + workflow, + tool=self._build_tool(provider), ) + for workflow, provider in rows + ) + + def _build_tool(self, provider: WorkflowToolProvider) -> WorkflowTool: + runtime = self._workflow_runtime + if runtime is None: + raise ValueError("Workflow execution dependencies are required to materialize Workflow Tools") + controller = WorkflowToolProviderController.from_db( + provider, + queries=runtime.tools, + draft_variable_saver=self._draft_variable_saver, + workflow_runtime=runtime, + ) + tools = controller.get_tools(provider.tenant_id) + if len(tools) != 1: + raise ValueError(f"Workflow Tool provider {provider.id} did not materialize exactly one tool") + return tools[0] @staticmethod def _materialize(workflow: Workflow, *, tool: WorkflowTool | None = None) -> WorkflowToolSource: diff --git a/api/services/agent/__init__.py b/api/services/agent/__init__.py index c95169198a764a..795367c0d4ebce 100644 --- a/api/services/agent/__init__.py +++ b/api/services/agent/__init__.py @@ -1 +1 @@ -"""Agent application services.""" +"""Agent use cases; import each service explicitly from its owning module.""" diff --git a/api/services/agent/chat/base_runner.py b/api/services/agent/chat/base_runner.py index a21a5e9546efbc..e6451ac4d1dcaa 100644 --- a/api/services/agent/chat/base_runner.py +++ b/api/services/agent/chat/base_runner.py @@ -12,7 +12,10 @@ from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.app.entities.app_invoke_entities import ( AgentChatAppGenerateEntity, + DifyRunContext, + InvokeFrom, ModelConfigWithCredentialsEntity, + UserFrom, ) from core.app.file_access import DatabaseFileAccessController from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler @@ -148,6 +151,15 @@ def _convert_tool_to_prompt_message_tool(self, tool: AgentToolEntity) -> tuple[P workflow_queries=self._workflow_runtime.tools, workflow_runtime=self._workflow_runtime, ) + invoke_from = self.application_generate_entity.invoke_from + if isinstance(invoke_from, InvokeFrom): + tool_entity.runtime.dify_run_context = DifyRunContext( + tenant_id=self.tenant_id, + app_id=self.app_config.app_id, + user_id=self.user_id, + user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER), + invoke_from=invoke_from, + ) assert tool_entity.entity.description message_tool = PromptMessageTool( name=tool.tool_name, diff --git a/api/services/agent/chat/cot_runner.py b/api/services/agent/chat/cot_runner.py index 176b01ab4f28cb..a19d70d2de909a 100644 --- a/api/services/agent/chat/cot_runner.py +++ b/api/services/agent/chat/cot_runner.py @@ -1,8 +1,8 @@ -import contextlib import json import logging from abc import ABC, abstractmethod from collections.abc import Generator, Mapping, Sequence +from contextlib import suppress from typing import Any from core.agent.entities import AgentScratchpadUnit @@ -317,7 +317,7 @@ def _handle_invoke_action( return answer, ToolInvokeMeta.error_instance(answer) if isinstance(tool_call_args, str): - with contextlib.suppress(json.JSONDecodeError): + with suppress(json.JSONDecodeError): tool_call_args = json.loads(tool_call_args) # invoke tool diff --git a/api/services/agent/composer_service.py b/api/services/agent/composer_service.py index 2ece84a96c6ec3..942b53fbdbd5a8 100644 --- a/api/services/agent/composer_service.py +++ b/api/services/agent/composer_service.py @@ -4,13 +4,15 @@ from sqlalchemy import func, or_, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot +from enums.agent import WorkflowAgentBindingType from libs.helper import to_timestamp from models import Account, App, Conversation from models.agent import ( APP_BACKED_AGENT_SOURCES, + WORKFLOW_EXECUTION_BINDING_VERSION, Agent, AgentConfigDraft, AgentConfigDraftType, @@ -25,16 +27,17 @@ AgentSource, AgentStatus, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import DeclaredOutputConfig +from models.agent_config_entities import DeclaredOutputConfig, agent_soul_has_model from models.agent_config_entities import ( effective_declared_outputs as _effective_declared_outputs, ) from models.workflow import Workflow +from repositories.agent.config_repository import AgentConfigRepository +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository, workflow_binding_scope -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.composer_validator import ComposerConfigValidator from services.agent.dsl_entities import make_portable_agent_soul from services.agent.errors import ( @@ -55,7 +58,7 @@ list_missing_tenant_knowledge_dataset_ids, ) from services.agent.knowledge_spaces import sync_agent_app_knowledge_bindings, validate_agent_knowledge_spaces -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.agent.retirement_service import WorkflowAgentRetirementService from services.agent.roster_service import AgentRosterService from services.agent.workspace_service import AgentWorkspaceNotFoundError, AgentWorkspaceService, WorkspaceOwnerScope from services.app_service import AppService, CreateAppParams @@ -68,6 +71,8 @@ WorkflowNodeJobConfig, ) from services.skill_management_service import SkillManagementService +from services.tools.provider_queries import ToolProviders +from services.workflow.variable_service import WorkflowVariableService from tasks.collect_agent_resources_task import enqueue_agent_resource_collection # WorkflowAgentNodeBinding.workflow_version tag for the draft workflow row. @@ -137,7 +142,7 @@ def load_workflow_composer( raise AgentVersionNotFoundError() return cls._empty_workflow_state(app_id=app_id, workflow_id=workflow.id, node_id=node_id) - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) version = cls._workflow_composer_version( session=session, tenant_id=tenant_id, @@ -180,14 +185,16 @@ def _workflow_composer_version( raise AgentVersionNotFoundError() else: raise AgentVersionNotFoundError() - return cls._require_version(session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=snapshot_id) + return AgentConfigRepository.require_snapshot( + session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=snapshot_id + ) version_id = ( agent.active_config_snapshot_id if agent and binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT else binding.current_snapshot_id ) - return cls._get_version_if_present( + return AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -299,7 +306,9 @@ def save_workflow_composer( ) state["validation"] = cls.collect_validation_findings(payload=payload) session.commit() - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=tenant_id, agent_ids=retirement_candidates, account_id=account_id, @@ -325,8 +334,8 @@ def copy_workflow_composer_from_roster( binding = cls._require_binding(cls._lock_workflow_binding(session=session, workflow=workflow, node_id=node_id)) if binding.binding_type == WorkflowAgentBindingType.INLINE_AGENT and idempotency_key: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - version = cls._get_version_if_present( + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -351,7 +360,7 @@ def copy_workflow_composer_from_roster( raise InvalidComposerConfigError("Source agent must be an active roster agent.") if not agent_has_workflow_callable_active_snapshot(session=session, agent=source_agent): raise InvalidComposerConfigError("Source agent must have a published config snapshot.") - source_version = cls._require_version( + source_version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=source_agent.id, @@ -390,7 +399,7 @@ def copy_workflow_composer_from_roster( binding.updated_by = account_id session.flush() - version = cls._require_version( + version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=inline_agent.id, @@ -491,7 +500,7 @@ def _load_agent_composer_for_agent( draft = ( None if published_only - else cls._get_agent_draft( + else AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -499,7 +508,7 @@ def _load_agent_composer_for_agent( account_id=None, ) ) - version = cls._get_version_if_present( + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, version_id=agent.active_config_snapshot_id ) if draft is None or ( @@ -688,7 +697,7 @@ def _agent_soul_matches_active_config( if not agent.active_config_snapshot_id: return False - active_version = cls._get_version_if_present( + active_version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -711,7 +720,7 @@ def publish_agent_app_draft( if agent.scope != AgentScope.ROSTER or agent.source not in APP_BACKED_AGENT_SOURCES: raise AgentNotFoundError() access_was_ready = agent_has_workflow_callable_active_snapshot(session=session, agent=agent) - draft = cls._get_or_create_agent_draft( + draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -826,7 +835,7 @@ def _checkout_agent_app_build_draft_in_transaction( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str, force: bool ) -> tuple[dict[str, Any], str | None]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - normal_draft = cls._get_or_create_agent_draft( + normal_draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -834,7 +843,7 @@ def _checkout_agent_app_build_draft_in_transaction( account_id=None, created_by=account_id, ) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -905,7 +914,7 @@ def _validate_active_build_draft_binding( ) if binding is None or binding.agent_id != agent.id: raise AgentBuildSandboxNotFoundError() - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=build_draft.home_snapshot_id, agent_config_version_id=build_draft.id, @@ -916,7 +925,7 @@ def _validate_active_build_draft_binding( def load_agent_app_build_draft( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str ) -> dict[str, Any]: - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent_id, @@ -984,7 +993,7 @@ def _apply_agent_app_build_draft_in_transaction( account_id: str, ) -> tuple[dict[str, Any], list[str]]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -1094,7 +1103,7 @@ def _retire_normal_preview_bindings( ) if binding is None or binding.agent_id != agent.id: raise AgentWorkspaceNotFoundError("Agent Preview participant Binding is unavailable") - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=normal_draft.home_snapshot_id, agent_config_version_id=normal_draft.id, @@ -1138,7 +1147,7 @@ def _discard_agent_app_build_draft_in_transaction( cls, *, session: Session, tenant_id: str, agent_id: str, account_id: str ) -> tuple[dict[str, Any], str | None]: agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) - build_draft = cls._get_agent_draft( + build_draft = AgentConfigRepository.get_draft( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -1222,7 +1231,15 @@ def resolve_workflow_node_agent_id( @classmethod def get_workflow_candidates( - cls, *, session: Session, tenant_id: str, app_id: str, node_id: str, user_id: str + cls, + *, + session: Session, + tenant_id: str, + app_id: str, + node_id: str, + user_id: str, + variables: WorkflowVariableService, + tool_providers: ToolProviders, ) -> dict[str, Any]: """Slash-menu data source for the workflow Agent node composer (ENG-615).""" from services.agent.composer_candidates import previous_node_output_candidates, soul_candidates @@ -1252,10 +1269,10 @@ def get_workflow_candidates( session=session, tenant_id=tenant_id, workflow_id=workflow.id, node_id=nid ), draft_variables_loader=lambda nid: cls._draft_node_variables( - session=session, app_id=app_id, node_id=nid, user_id=user_id + variables=variables, app_id=app_id, node_id=nid, user_id=user_id ), system_variables_loader=lambda: cls._draft_system_variables( - session=session, app_id=app_id, user_id=user_id + variables=variables, app_id=app_id, user_id=user_id ), ) truncated = truncated or outputs_truncated @@ -1265,7 +1282,9 @@ def get_workflow_candidates( dataset_lookup=lambda ids: get_tenant_knowledge_dataset_rows( session=session, tenant_id=tenant_id, dataset_ids=ids ), - workspace_tools_loader=lambda: cls._workspace_dify_tools(tenant_id=tenant_id, user_id=user_id), + workspace_tools_loader=lambda: cls._workspace_dify_tools( + tenant_id=tenant_id, user_id=user_id, tool_providers=tool_providers + ), ) truncated = truncated or soul_truncated @@ -1285,7 +1304,7 @@ def get_workflow_candidates( @classmethod def get_agent_app_candidates( - cls, *, session: Session, tenant_id: str, agent_id: str, user_id: str + cls, *, session: Session, tenant_id: str, agent_id: str, user_id: str, tool_providers: ToolProviders ) -> dict[str, Any]: """Slash-menu data source for the Agent App (Console) composer (ENG-615).""" from services.agent.composer_candidates import soul_candidates @@ -1296,7 +1315,9 @@ def get_agent_app_candidates( dataset_lookup=lambda ids: get_tenant_knowledge_dataset_rows( session=session, tenant_id=tenant_id, dataset_ids=ids ), - workspace_tools_loader=lambda: cls._workspace_dify_tools(tenant_id=tenant_id, user_id=user_id), + workspace_tools_loader=lambda: cls._workspace_dify_tools( + tenant_id=tenant_id, user_id=user_id, tool_providers=tool_providers + ), ) response = ComposerCandidatesResponse( variant=ComposerVariant.AGENT_APP, @@ -1320,8 +1341,8 @@ def _parse_node_job(binding: WorkflowAgentNodeBinding) -> WorkflowNodeJobConfig def _load_binding_soul( cls, *, session: Session, tenant_id: str, binding: WorkflowAgentNodeBinding ) -> AgentSoulConfig | None: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - version = cls._get_version_if_present( + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) + version = AgentConfigRepository.get_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id if agent else None, @@ -1331,10 +1352,10 @@ def _load_binding_soul( @classmethod def _load_agent_soul(cls, *, session: Session, tenant_id: str, agent_id: str) -> AgentSoulConfig | None: - agent = cls._get_agent_if_present(session=session, tenant_id=tenant_id, agent_id=agent_id) + agent = AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) if agent is None: return None - draft = cls._get_or_create_agent_draft( + draft = AgentConfigRepository.get_or_create_draft( session=session, tenant_id=tenant_id, agent=agent, @@ -1369,30 +1390,30 @@ def _binding_declared_outputs( return list(_effective_declared_outputs(node_job.declared_outputs, node_job.output_routes)) @staticmethod - def _draft_node_variables(*, session: Any, app_id: str, node_id: str, user_id: str) -> list[tuple[str, str | None]]: - from services.workflow_draft_variable_service import WorkflowDraftVariableService - - variables = WorkflowDraftVariableService(session=session).list_node_variables(app_id, node_id, user_id) - return [(variable.name, variable.value_type.value) for variable in variables.variables] + def _draft_node_variables( + *, variables: WorkflowVariableService, app_id: str, node_id: str, user_id: str + ) -> list[tuple[str, str | None]]: + result = variables.list_node_variables(app_id, node_id, user_id) + return [(variable.name, variable.value_type.value) for variable in result.variables] @staticmethod - def _draft_system_variables(*, session: Any, app_id: str, user_id: str) -> list[tuple[str, str | None]]: - from services.workflow_draft_variable_service import WorkflowDraftVariableService - - variables = WorkflowDraftVariableService(session=session).list_system_variables(app_id, user_id) - return [(variable.name, variable.value_type.value) for variable in variables.variables] + def _draft_system_variables( + *, variables: WorkflowVariableService, app_id: str, user_id: str + ) -> list[tuple[str, str | None]]: + result = variables.list_system_variables(app_id, user_id) + return [(variable.name, variable.value_type.value) for variable in result.variables] @staticmethod - def _workspace_dify_tools(*, tenant_id: str, user_id: str) -> list[dict[str, Any]]: + def _workspace_dify_tools(*, tenant_id: str, user_id: str, tool_providers: ToolProviders) -> list[dict[str, Any]]: """Workspace Dify Plugin tools, same source as the tool selector. A plugin-daemon outage must degrade the slash menu to an empty tools tab, not break the whole candidates endpoint. """ - from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService + from services.tools.builtin_tools_manage_service import BuiltinToolManageService try: - providers = BuiltinToolManageService.list_builtin_tools(user_id, tenant_id) + providers = BuiltinToolManageService.list_builtin_tools(user_id, tenant_id, tool_providers=tool_providers) except Exception: logger.warning("candidates: failed to list workspace tools for tenant %s", tenant_id, exc_info=True) return [] @@ -1450,6 +1471,7 @@ def calculate_impact(cls, *, session: Session, tenant_id: str, current_snapshot_ select(WorkflowAgentNodeBinding).where( WorkflowAgentNodeBinding.tenant_id == tenant_id, or_(*predicates), + WorkflowAgentNodeBinding.workflow_version != WORKFLOW_EXECUTION_BINDING_VERSION, ) ).all() ) @@ -1637,7 +1659,7 @@ def _save_to_current_version( binding = cls._require_binding(binding) if payload.agent_soul is None: raise ValueError("agent_soul is required") - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=binding.agent_id, @@ -1689,7 +1711,7 @@ def _save_as_new_version( binding = cls._require_binding(binding) if not binding.agent_id or payload.agent_soul is None: raise ValueError("agent_id and agent_soul are required") - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=binding.agent_id, @@ -1804,7 +1826,7 @@ def _save_to_roster( ) -> WorkflowAgentNodeBinding: binding = cls._require_binding(binding) source_agent = cls._require_agent(session=session, tenant_id=tenant_id, agent_id=binding.agent_id) - source_version = cls._require_version( + source_version = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=source_agent.id, @@ -1948,7 +1970,7 @@ def _create_roster_agent_for_composer( if agent is None: raise AgentNotFoundError() - current_snapshot = cls._require_version( + current_snapshot = AgentConfigRepository.require_snapshot( session=session, tenant_id=tenant_id, agent_id=agent.id, @@ -2113,10 +2135,7 @@ def get_or_create_normal_agent_draft( agent: Agent, created_by: str | None, ) -> AgentConfigDraft: - """Resolve the normal Draft, rebasing only stale WORKFLOW_ONLY DRAFT rows whose account_id is None. - - Roster and DEBUG_BUILD Drafts are never rebased. - """ + """Resolve the normal Draft, rebasing only stale workflow-owned rows.""" return cls._get_or_create_agent_draft( session=session, tenant_id=tenant_id, @@ -2366,43 +2385,28 @@ def _require_account(cls, *, session: Session, account_id: str) -> Account: @classmethod def _get_agent_if_present(cls, *, session: Session, tenant_id: str, agent_id: str | None) -> Agent | None: - if not agent_id: - return None - return session.scalar(select(Agent).where(Agent.tenant_id == tenant_id, Agent.id == agent_id).limit(1)) + return AgentConfigRepository.get_agent(session=session, tenant_id=tenant_id, agent_id=agent_id) @classmethod def _require_version( cls, *, session: Session, tenant_id: str, agent_id: str | None, version_id: str | None ) -> AgentConfigSnapshot: - if not agent_id or not version_id: - raise AgentVersionNotFoundError() - version = session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent_id, - AgentConfigSnapshot.id == version_id, - ) - .limit(1) + return AgentConfigRepository.require_snapshot( + session=session, + tenant_id=tenant_id, + agent_id=agent_id, + version_id=version_id, ) - if not version: - raise AgentVersionNotFoundError() - return version @classmethod def _get_version_if_present( cls, *, session: Session, tenant_id: str, agent_id: str | None, version_id: str | None ) -> AgentConfigSnapshot | None: - if not agent_id or not version_id: - return None - return session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent_id, - AgentConfigSnapshot.id == version_id, - ) - .limit(1) + return AgentConfigRepository.get_snapshot( + session=session, + tenant_id=tenant_id, + agent_id=agent_id, + version_id=version_id, ) @staticmethod diff --git a/api/services/agent/dsl_entities.py b/api/services/agent/dsl_entities.py index 6fa9392c911caf..b6b6adb9e77664 100644 --- a/api/services/agent/dsl_entities.py +++ b/api/services/agent/dsl_entities.py @@ -9,7 +9,7 @@ from core.plugin.entities.plugin import PluginDependency from models.agent import Agent -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.model import App from services.entities.dsl_entities import make_app_dsl from services.entities.site_dsl import SiteDsl @@ -19,17 +19,6 @@ AGENT_NODE_JOB_DSL_KEY = "agent_job" -class AgentPackageMetadata(BaseModel): - model_config = ConfigDict(extra="forbid") - - name: str = Field(min_length=1, max_length=255) - description: str = "" - role: str = "" - icon_type: str | None = None - icon: str | None = None - icon_background: str | None = None - - class AgentPackageOmittedAsset(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/api/services/agent/dsl_service.py b/api/services/agent/dsl_service.py index 5012f54e0eafcf..77d88af80ce03e 100644 --- a/api/services/agent/dsl_service.py +++ b/api/services/agent/dsl_service.py @@ -9,17 +9,15 @@ from __future__ import annotations import copy -import json from collections.abc import Mapping from typing import Any, cast from uuid import UUID from pydantic import BaseModel, ConfigDict, Field -from sqlalchemy import func, select, update +from sqlalchemy import select, update from sqlalchemy.orm import Session -from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidator +from enums.agent import WorkflowAgentBindingType from libs.datetime_utils import naive_utc_now from models import Account from models.agent import ( @@ -27,29 +25,25 @@ Agent, AgentConfigDraft, AgentConfigDraftType, - AgentConfigRevision, AgentConfigRevisionOperation, AgentConfigSnapshot, AgentIconType, - AgentKind, AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig, WorkflowNodeJobConfig from models.model import App, AppModelConfig, UploadFile from models.skill import AgentSkillBinding, AgentSkillBindingSnapshot, Skill from models.tools import ToolFile from models.workflow import Workflow -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent.creation_repository import WorkflowAgentCreationRepository from services.agent.dependency_service import extract_agent_soul_dependencies from services.agent.dsl_entities import ( AGENT_NODE_JOB_DSL_KEY, AGENT_PACKAGE_REF_KEY, AgentPackage, - AgentPackageMetadata, AgentPackageWorkspaceSkill, make_portable_agent_package, portable_ref, @@ -57,7 +51,9 @@ from services.agent.knowledge_datasets import get_tenant_knowledge_dataset_rows from services.agent.package_resource_exporter import AgentPackageResourceExporter from services.agent.roster_service import AgentRosterService +from services.agent.workflow_validator import WorkflowAgentNodeValidator from services.entities.dsl_entities import DslImportWarning +from services.workflow.contracts import WorkflowBindingScope class AgentPackageImportResult(BaseModel): @@ -288,12 +284,16 @@ def import_agent_app_package( def import_workflow_packages( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, portable_graph: Mapping[str, Any], raw_packages: Mapping[str, Any], account: Account, ) -> tuple[dict[str, Any], list[DslImportWarning], set[str]]: - """Materialize every packaged Agent as a node-owned inline Agent.""" + """Stage node-owned Agent packages and return the resolved graph. + + The draft writer persists the returned graph in this same transaction, + keeping the workflow revision and imported bindings atomic. + """ graph = copy.deepcopy(dict(portable_graph)) packages = {key: AgentPackage.model_validate(value) for key, value in raw_packages.items()} @@ -364,41 +364,9 @@ def import_workflow_packages( node_data.pop(AGENT_NODE_JOB_DSL_KEY, None) warnings.extend(imported.warnings) - workflow.graph = json.dumps(graph) self.session.flush() return graph, warnings, retirement_candidates - def clone_inline_binding_for_node( - self, - *, - workflow: Workflow, - node_id: str, - source_agent: Agent, - source_snapshot: AgentConfigSnapshot, - account_id: str, - ) -> tuple[Agent, AgentConfigSnapshot]: - """Clone a same-workspace Inline Agent for a pasted target node.""" - - soul = AgentSoulConfig.model_validate(source_snapshot.config_snapshot_dict) - metadata = AgentPackageMetadata( - name=source_agent.name, - description=source_agent.description, - role=source_agent.role, - icon_type=source_agent.icon_type.value if source_agent.icon_type else None, - icon=source_agent.icon, - icon_background=source_agent.icon_background, - ) - agent, snapshot = self._create_workflow_only_agent( - workflow=workflow, - node_id=node_id, - account_id=account_id, - metadata=metadata, - soul=soul, - source=AgentSource.WORKFLOW, - operation=AgentConfigRevisionOperation.CREATE_VERSION, - ) - return agent, snapshot - def extract_package_dependencies(self, packages: Mapping[str, AgentPackage]) -> list[str]: dependencies: list[str] = [] for package in packages.values(): @@ -504,7 +472,7 @@ def _restore_workspace_skill_bindings( def _create_imported_inline_agent( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, node_id: str, account: Account, package: AgentPackage, @@ -546,7 +514,7 @@ def _create_imported_inline_agent( def _create_workflow_only_agent( self, *, - workflow: Workflow, + workflow: WorkflowBindingScope, node_id: str, account_id: str, metadata: AgentPackageMetadata, @@ -554,48 +522,15 @@ def _create_workflow_only_agent( source: AgentSource, operation: AgentConfigRevisionOperation, ) -> tuple[Agent, AgentConfigSnapshot]: - backing_app = AgentRosterService(self.session).create_hidden_backing_app_for_workflow_agent( - tenant_id=workflow.tenant_id, - account_id=account_id, - name=metadata.name, - description=metadata.description, - icon_type=metadata.icon_type, - icon=metadata.icon, - icon_background=metadata.icon_background, - ) - agent = Agent( - tenant_id=workflow.tenant_id, - name=metadata.name, - description=metadata.description, - role=metadata.role, - icon_type=self._agent_icon_type(metadata.icon_type), - icon=metadata.icon, - icon_background=metadata.icon_background, - agent_kind=AgentKind.DIFY_AGENT, - scope=AgentScope.WORKFLOW_ONLY, - source=source, - app_id=workflow.app_id, - backing_app_id=backing_app.id, - workflow_id=workflow.id, - workflow_node_id=node_id, - status=AgentStatus.ACTIVE, - created_by=account_id, - updated_by=account_id, - ) - self.session.add(agent) - self.session.flush() - snapshot = self._create_snapshot( - tenant_id=workflow.tenant_id, - agent=agent, + return WorkflowAgentCreationRepository(self.session).create_workflow_agent( + workflow=workflow, + node_id=node_id, account_id=account_id, + metadata=metadata, soul=soul, + source=source, operation=operation, ) - agent.active_config_snapshot_id = snapshot.id - agent.active_config_has_model = agent_soul_has_model(soul) - agent.active_config_is_published = True - self.session.flush() - return agent, snapshot def resolve_package_soul( self, @@ -738,46 +673,6 @@ def _mark_missing_package_assets( ) return warnings - def _create_snapshot( - self, - *, - tenant_id: str, - agent: Agent, - account_id: str, - soul: AgentSoulConfig, - operation: AgentConfigRevisionOperation, - ) -> AgentConfigSnapshot: - next_version = ( - self.session.scalar( - select(func.max(AgentConfigSnapshot.version)).where( - AgentConfigSnapshot.tenant_id == tenant_id, - AgentConfigSnapshot.agent_id == agent.id, - ) - ) - or 0 - ) + 1 - snapshot = AgentConfigSnapshot( - tenant_id=tenant_id, - agent_id=agent.id, - version=next_version, - config_snapshot=soul, - home_snapshot_id=None, - created_by=account_id, - ) - self.session.add(snapshot) - self.session.flush() - revision = AgentConfigRevision( - tenant_id=tenant_id, - agent_id=agent.id, - current_snapshot_id=snapshot.id, - revision=1, - operation=operation, - created_by=account_id, - ) - self.session.add(revision) - self.session.flush() - return snapshot - def unique_roster_name(self, *, tenant_id: str, requested: str) -> str: candidates = [requested] for index in range(1, 100): @@ -820,11 +715,3 @@ def _require_snapshot(self, *, tenant_id: str, agent_id: str, snapshot_id: str | @staticmethod def _agent_icon_type(value: str | None) -> AgentIconType | None: return AgentIconType(value) if value else None - - -def is_agent_v2_graph(graph: Mapping[str, Any]) -> bool: - return any( - isinstance(node.get("data"), Mapping) and is_dify_agent_node_data(node["data"]) - for node in graph.get("nodes", []) - if isinstance(node, Mapping) - ) diff --git a/api/services/agent/home_snapshot_service.py b/api/services/agent/home_snapshot_service.py index 707a379602881a..579cb55c48f8a3 100644 --- a/api/services/agent/home_snapshot_service.py +++ b/api/services/agent/home_snapshot_service.py @@ -25,6 +25,7 @@ AgentWorkingResourceStatus, AgentWorkspaceOwnerType, ) +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent.errors import ( AgentBuildSandboxNotFoundError, AgentHomeSnapshotCreateFailedError, @@ -40,7 +41,7 @@ class AgentHomeSnapshotUnavailableError(RuntimeError): class AgentHomeSnapshotService: - """Create, retire, and collect Agent-owned immutable Home Snapshots.""" + """Create and collect Agent-owned immutable Home Snapshots.""" @classmethod def create_for_build_apply( @@ -80,7 +81,7 @@ def create_for_build_apply( ) if binding is None or binding.agent_id != build_draft.agent_id: raise AgentBuildSandboxNotFoundError() - AgentWorkspaceService.validate_binding_generation( + AgentWorkspaceRepository.validate_binding_generation( binding, base_home_snapshot_id=build_draft.home_snapshot_id, agent_config_version_id=build_draft.id, @@ -121,21 +122,6 @@ def create_for_build_apply( session.add(home_snapshot) return home_snapshot - @classmethod - def retire_all_for_agent(cls, *, session: Session, tenant_id: str, agent_id: str) -> list[str]: - rows = session.scalars( - select(AgentHomeSnapshot).where( - AgentHomeSnapshot.tenant_id == tenant_id, - AgentHomeSnapshot.agent_id == agent_id, - ) - ).all() - now = naive_utc_now() - for row in rows: - if row.status == AgentWorkingResourceStatus.ACTIVE: - row.status = AgentWorkingResourceStatus.RETIRED - row.retired_at = now - return [row.id for row in rows] - @classmethod def retire_all_for_tenant(cls, *, session: Session, tenant_id: str) -> list[str]: """Retire every tenant Home Snapshot in the caller's transaction. diff --git a/api/services/agent/knowledge_spaces.py b/api/services/agent/knowledge_spaces.py index dcbc0b9716d90e..51205146bde2b1 100644 --- a/api/services/agent/knowledge_spaces.py +++ b/api/services/agent/knowledge_spaces.py @@ -9,9 +9,10 @@ from __future__ import annotations from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.db.session_factory import session_factory +from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data from models.agent import Agent, AgentConfigSnapshot, WorkflowAgentNodeBinding from models.agent_config_entities import AgentSoulConfig from models.knowledge_fs import ( @@ -22,6 +23,7 @@ ) from models.workflow import Workflow from services.agent.errors import InvalidComposerConfigError +from services.knowledge_fs.knowledge_retrieval_v2.validation import collect_control_space_ids from services.knowledge_fs.product_authorization import ( DifyKnowledgeFSProductRBACPort, effective_product_permissions, @@ -142,3 +144,54 @@ def collect_workflow_agent_knowledge_space_ids(*, session: Session, workflow: Wo for binding in AgentSoulConfig.model_validate(snapshot.config_snapshot_dict).knowledge.spaces ) ) + + +def sync_workflow_app_knowledge_bindings( + *, + session: Session, + session_maker: sessionmaker[Session], + workflow: Workflow, +) -> None: + """Reconcile a published workflow's direct and frozen-Agent KnowledgeFS grants atomically.""" + graph = workflow.graph_dict + control_space_ids = collect_control_space_ids(graph) + if any( + isinstance(node, dict) and isinstance(node.get("data"), dict) and is_dify_agent_node_data(node["data"]) + for node in graph.get("nodes", []) + ): + # Published Agent bindings are inserted immediately before this policy runs. + # Flush them so the grant union cannot omit an Agent from this transaction. + session.flush() + control_space_ids = tuple( + dict.fromkeys( + ( + *control_space_ids, + *collect_workflow_agent_knowledge_space_ids(session=session, workflow=workflow), + ) + ) + ) + + if ( + not control_space_ids + and session.scalar( + select(AppKnowledgeFSSpaceJoin.id) + .where( + AppKnowledgeFSSpaceJoin.tenant_id == workflow.tenant_id, + AppKnowledgeFSSpaceJoin.app_id == workflow.app_id, + AppKnowledgeFSSpaceJoin.join_type == KnowledgeFSAppSpaceJoinType.WORKFLOW, + ) + .limit(1) + ) + is None + ): + return + + from services.knowledge_fs.runtime import get_knowledge_fs_runtime + + get_knowledge_fs_runtime(session_maker).app_bindings.sync_workflow_bindings( + tenant_id=workflow.tenant_id, + actor_account_id=workflow.created_by, + app_id=workflow.app_id, + control_space_ids=list(control_space_ids), + session=session, + ) diff --git a/api/services/agent/legacy_retirement_service.py b/api/services/agent/legacy_retirement_service.py index c800a8ea51bd58..ef3b67e5862a89 100644 --- a/api/services/agent/legacy_retirement_service.py +++ b/api/services/agent/legacy_retirement_service.py @@ -21,7 +21,7 @@ ) from models.model import App, AppMode from models.workflow import Workflow -from services.agent.home_snapshot_service import AgentHomeSnapshotService +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository from services.agent.workspace_service import AgentWorkspaceService from tasks.collect_agent_resources_task import enqueue_agent_resource_collection from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -105,8 +105,7 @@ def retire_unowned( ) retired_bindings.append(binding.id) retired_snapshots.extend( - AgentHomeSnapshotService.retire_all_for_agent( - session=session, + AgentHomeSnapshotRepository(session).retire_all_for_agent( tenant_id=tenant_id, agent_id=agent_id, ) diff --git a/api/services/agent/legacy_workflow_publish_service.py b/api/services/agent/legacy_workflow_publish_service.py index 0e8e866da256c8..c28a1090bd77aa 100644 --- a/api/services/agent/legacy_workflow_publish_service.py +++ b/api/services/agent/legacy_workflow_publish_service.py @@ -25,6 +25,7 @@ WorkflowPreviousNodeOutputRef, ) from models.workflow import Workflow +from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository, workflow_binding_scope from services.agent.composer_validator import ComposerConfigValidator from services.agent.prompt_mentions import ( extract_workflow_node_output_selectors, @@ -446,40 +447,40 @@ def _clone_inline_graph_binding_for_node( source_snapshot_id: str, account_id: str, ) -> tuple[Agent, str]: - source_agent = session.scalar( - select(Agent) - .where( - Agent.tenant_id == draft_workflow.tenant_id, - Agent.id == source_agent_id, - Agent.scope == AgentScope.WORKFLOW_ONLY, - Agent.status == AgentStatus.ACTIVE, - ) - .limit(1) - ) - if source_agent is None: + binding_repository = WorkflowAgentBindingRepository(session) + source_agent = binding_repository.get_agent(draft_workflow.tenant_id, source_agent_id) + if ( + source_agent is None + or source_agent.scope != AgentScope.WORKFLOW_ONLY + or source_agent.status != AgentStatus.ACTIVE + ): raise ValueError(f"Workflow Agent node {node_id} references an unavailable inline agent.") - source_snapshot = session.scalar( - select(AgentConfigSnapshot) - .where( - AgentConfigSnapshot.tenant_id == draft_workflow.tenant_id, - AgentConfigSnapshot.agent_id == source_agent.id, - AgentConfigSnapshot.id == source_snapshot_id, - ) - .limit(1) + source_snapshot = binding_repository.get_snapshot( + draft_workflow.tenant_id, + source_agent.id, + source_snapshot_id, ) if source_snapshot is None: raise ValueError(f"Workflow Agent node {node_id} references a missing inline agent config snapshot.") - from services.agent.dsl_service import AgentDslService - - agent, snapshot = AgentDslService(session).clone_inline_binding_for_node( - workflow=draft_workflow, + cloned_agent, snapshot_id = binding_repository.clone( + workflow=workflow_binding_scope(draft_workflow), node_id=node_id, source_agent=source_agent, source_snapshot=source_snapshot, account_id=account_id, ) - return agent, snapshot.id + agent = session.scalar( + select(Agent) + .where( + Agent.tenant_id == draft_workflow.tenant_id, + Agent.id == cloned_agent.id, + ) + .limit(1) + ) + if agent is None: + raise RuntimeError(f"Workflow Agent clone for node {node_id} was not persisted.") + return agent, snapshot_id @classmethod def _resolve_roster_agent_graph_binding( diff --git a/api/services/agent/roster_package_importer.py b/api/services/agent/roster_package_importer.py index 54bbd05b57db07..f292424de890c3 100644 --- a/api/services/agent/roster_package_importer.py +++ b/api/services/agent/roster_package_importer.py @@ -27,9 +27,10 @@ AgentIconType, AgentSource, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.model import App, AppMode, AppModelConfig, IconType, UploadFile -from services.agent.dsl_entities import AgentAppDsl, AgentPackage, AgentPackageMetadata +from repositories.app.creation_records import create_installed_app_record, create_site_record +from services.agent.dsl_entities import AgentAppDsl, AgentPackage from services.agent.dsl_service import AgentDslService from services.agent.errors import ( AgentNameConflictError, @@ -43,7 +44,6 @@ from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.agent.roster_package_reader import RosterAgentPackageReader from services.agent.roster_service import AgentRosterService -from services.app_creation_records import create_installed_app_record, create_site_record from services.app_dsl_service import AppDslService from services.app_import_source import download_app_import_source from services.app_service import AppService diff --git a/api/services/agent/roster_service.py b/api/services/agent/roster_service.py index af70071736ebae..8d6951fee23a5a 100644 --- a/api/services/agent/roster_service.py +++ b/api/services/agent/roster_service.py @@ -6,8 +6,8 @@ from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError -from constants.model_template import default_app_templates from core.agent.publish_visibility import workflow_callable_active_snapshot_filter +from enums.agent import WorkflowAgentBindingType from libs.datetime_utils import naive_utc_now from libs.helper import to_timestamp from models.agent import ( @@ -26,14 +26,13 @@ AgentSource, AgentStatus, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentSoulConfig, agent_soul_has_model from models.enums import AppStatus, ConversationFromSource, ConversationStatus, InvokeFrom -from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message +from models.model import App, AppMode, Conversation, IconType, Message from models.workflow import Workflow -from services.agent.agent_soul_state import agent_soul_has_model +from repositories.agent.creation_repository import WorkflowAgentCreationRepository from services.agent.composer_validator import ComposerConfigValidator from services.agent.errors import ( AgentArchivedError, @@ -474,33 +473,15 @@ def create_hidden_backing_app_for_workflow_agent( workspace Agent Roster until explicitly saved to roster. """ - app_template = dict(default_app_templates[AppMode.AGENT]["app"]) - app = App(**app_template) - app.name = name - app.description = description or "" - app.mode = AppMode.AGENT - normalized_icon_type = self._normalize_app_icon_type(icon_type) - app.icon_type = IconType(normalized_icon_type) if normalized_icon_type else IconType.EMOJI - app.icon = icon - app.icon_background = icon_background - app.tenant_id = tenant_id - app.enable_site = False - app.enable_api = False - app.api_rph = 0 - app.api_rpm = 0 - app.max_active_requests = None - app.created_by = account_id - app.maintainer = account_id - app.updated_by = account_id - self._session.add(app) - self._session.flush() - - app_model_config = AppModelConfig(app_id=app.id, created_by=account_id, updated_by=account_id) - self._session.add(app_model_config) - self._session.flush() - app.app_model_config_id = app_model_config.id - self._session.flush() - return app + return WorkflowAgentCreationRepository(self._session).create_hidden_backing_app( + tenant_id=tenant_id, + account_id=account_id, + name=name, + description=description, + icon_type=icon_type, + icon=icon, + icon_background=icon_background, + ) def _create_agent_app_debug_conversation(self, *, app_id: str, account_id: str) -> str: """Create one console debug conversation for an Agent App editor.""" @@ -1539,29 +1520,6 @@ def _next_revision(self, *, tenant_id: str, agent_id: str) -> int: or 0 ) + 1 - def _load_published_active_snapshot_agent_ids(self, *, tenant_id: str, agents: list[Agent]) -> set[str]: - predicates = [ - and_( - AgentConfigRevision.agent_id == agent.id, - AgentConfigRevision.current_snapshot_id == agent.active_config_snapshot_id, - AgentConfigRevision.operation.in_(self._visible_version_operations(agent)), - ) - for agent in agents - if agent.active_config_snapshot_id - ] - if not predicates: - return set() - - agent_ids = self._session.scalars( - select(AgentConfigRevision.agent_id) - .where( - AgentConfigRevision.tenant_id == tenant_id, - or_(*predicates), - ) - .distinct() - ).all() - return set(agent_ids) - def _load_published_references_by_agent_id( self, *, tenant_id: str, agent_ids: list[str] ) -> dict[str, list[AgentReferencingWorkflow]]: diff --git a/api/services/agent/workspace_service.py b/api/services/agent/workspace_service.py index 25732720cfe880..3e25cc12ca1fd5 100644 --- a/api/services/agent/workspace_service.py +++ b/api/services/agent/workspace_service.py @@ -24,7 +24,6 @@ WorkspaceOwnerScope, ) from core.db.session_factory import session_factory -from libs.datetime_utils import naive_utc_now from libs.uuid_utils import uuidv7 from models.agent import ( AgentConfigVersionKind, @@ -207,43 +206,7 @@ def save_binding_session_snapshot( @classmethod def retire_binding(cls, *, session: Session, tenant_id: str, binding_id: str) -> str | None: - binding = session.scalar( - select(AgentWorkspaceBinding) - .where( - AgentWorkspaceBinding.id == binding_id, - AgentWorkspaceBinding.tenant_id == tenant_id, - AgentWorkspaceBinding.status == AgentWorkingResourceStatus.ACTIVE, - ) - .with_for_update() - ) - if binding is None: - return None - workspace = session.scalar( - select(AgentWorkspace) - .where( - AgentWorkspace.id == binding.workspace_id, - AgentWorkspace.tenant_id == tenant_id, - AgentWorkspace.status == AgentWorkingResourceStatus.ACTIVE, - ) - .with_for_update() - ) - now = naive_utc_now() - binding.status = AgentWorkingResourceStatus.RETIRED - binding.retired_at = now - if workspace is not None: - other_binding = session.scalar( - select(AgentWorkspaceBinding.id).where( - AgentWorkspaceBinding.tenant_id == tenant_id, - AgentWorkspaceBinding.workspace_id == workspace.id, - AgentWorkspaceBinding.status == AgentWorkingResourceStatus.ACTIVE, - AgentWorkspaceBinding.id != binding.id, - ) - ) - if other_binding is None: - workspace.status = AgentWorkingResourceStatus.RETIRED - workspace.active_guard = None - workspace.retired_at = now - return binding.id + return AgentWorkspaceRepository(session=session).retire_binding(tenant_id=tenant_id, binding_id=binding_id) @classmethod def retire_workspace(cls, *, session: Session, tenant_id: str, workspace_id: str) -> str | None: @@ -466,14 +429,13 @@ def validate_binding_generation( agent_config_version_id: str, agent_config_version_kind: AgentConfigVersionKind, ) -> None: - if ( - binding.base_home_snapshot_id != base_home_snapshot_id - or binding.agent_config_version_id != agent_config_version_id - or binding.agent_config_version_kind != agent_config_version_kind - ): - raise AgentWorkspaceBindingGenerationMismatchError( - "ACTIVE Binding belongs to a different Agent config/Home generation" - ) + """Validate a legacy execution caller against the repository-owned generation policy.""" + AgentWorkspaceRepository.validate_binding_generation( + binding, + base_home_snapshot_id=base_home_snapshot_id, + agent_config_version_id=agent_config_version_id, + agent_config_version_kind=agent_config_version_kind, + ) @staticmethod def _client() -> Client: diff --git a/api/services/agent_service.py b/api/services/agent_service.py index bcc2539ab93457..cc7bfc3a6a809f 100644 --- a/api/services/agent_service.py +++ b/api/services/agent_service.py @@ -1,170 +1,9 @@ -import threading -from typing import Any - -import pytz -from sqlalchemy import select -from sqlalchemy.orm import Session - -import contexts -from core.app.app_config.easy_ui_based_app.agent.manager import AgentConfigManager from core.plugin.entities.plugin_daemon import PluginAgentProviderEntity from core.plugin.impl.agent import PluginAgentClient from core.plugin.impl.exc import PluginDaemonClientSideError -from core.tools.entities.tool_entities import EmojiIconDict -from core.tools.tool_manager import ToolManager -from libs.login import current_user -from models import Account -from models.model import App, Conversation, EndUser, Message, load_annotation_reply_config class AgentService: - @classmethod - def get_agent_logs(cls, app_model: App, conversation_id: str, message_id: str, session: Session): - """ - Service to get agent logs - """ - contexts.plugin_tool_providers.set({}) - contexts.plugin_tool_providers_lock.set(threading.Lock()) - - conversation: Conversation | None = session.scalar( - select(Conversation) - .where( - Conversation.id == conversation_id, - Conversation.app_id == app_model.id, - ) - .limit(1) - ) - - if not conversation: - raise ValueError(f"Conversation not found: {conversation_id}") - - message: Message | None = session.scalar( - select(Message) - .where( - Message.id == message_id, - Message.conversation_id == conversation_id, - ) - .limit(1) - ) - - if not message: - raise ValueError(f"Message not found: {message_id}") - - agent_thoughts = message.agent_thoughts_with_session(session=session) - - if conversation.from_end_user_id: - # only select name field - executor_name = session.scalar(select(EndUser.name).where(EndUser.id == conversation.from_end_user_id)) - else: - executor_name = session.scalar(select(Account.name).where(Account.id == conversation.from_account_id)) - - executor = executor_name or "Unknown" - assert isinstance(current_user, Account) - assert current_user.timezone is not None - timezone = pytz.timezone(current_user.timezone) - - app_model_config = app_model.app_model_config_with_session(session=session) - if not app_model_config: - raise ValueError("App model config not found") - - result: dict[str, Any] = { - "meta": { - "status": "success", - "executor": executor, - "start_time": message.created_at.astimezone(timezone).isoformat(), - "elapsed_time": message.provider_response_latency, - "total_tokens": message.answer_tokens + message.message_tokens, - "agent_mode": app_model_config.agent_mode_dict.get("strategy", "react"), - "iterations": len(agent_thoughts), - }, - "iterations": [], - "files": message.message_files_with_session(session=session), - } - - annotation_reply = load_annotation_reply_config(session, app_model.id) - agent_config = AgentConfigManager.convert(app_model_config.to_dict(annotation_reply=annotation_reply)) - if not agent_config: - raise ValueError("Agent config not found") - - agent_tools = agent_config.tools or [] - - def find_agent_tool(tool_name: str): - for agent_tool in agent_tools: - if agent_tool.tool_name == tool_name: - return agent_tool - - for agent_thought in agent_thoughts: - tools = agent_thought.tools - tool_labels = agent_thought.tool_labels - tool_meta = agent_thought.tool_meta - tool_inputs = agent_thought.tool_inputs_dict - tool_outputs = agent_thought.tool_outputs_dict or {} - tool_calls = [] - for tool in tools: - tool_name = tool - tool_label = tool_labels.get(tool_name, tool_name) - tool_input = tool_inputs.get(tool_name, {}) - tool_output = tool_outputs.get(tool_name, {}) - tool_meta_data = tool_meta.get(tool_name, {}) - tool_config = tool_meta_data.get("tool_config", {}) - tool_provider_type = tool_config.get("tool_provider_type", "") - tool_provider_id = tool_config.get("tool_provider", "") - - if not tool_provider_type: - tool_entity = find_agent_tool(tool_name) - if tool_entity: - tool_provider_type = tool_entity.provider_type - tool_provider_id = tool_provider_id or tool_entity.provider_id - - tool_icon: str | EmojiIconDict = "" - - if tool_provider_type and tool_provider_type != "dataset-retrieval": - tool_icon = ToolManager.get_tool_icon( - tenant_id=app_model.tenant_id, - provider_type=tool_provider_type, - provider_id=tool_provider_id, - ) - if not tool_icon: - tool_entity = find_agent_tool(tool_name) - if tool_entity: - tool_icon = ToolManager.get_tool_icon( - tenant_id=app_model.tenant_id, - provider_type=tool_entity.provider_type, - provider_id=tool_entity.provider_id, - ) - else: - tool_icon = "" - - tool_calls.append( - { - "status": "success" if not tool_meta_data.get("error") else "error", - "error": tool_meta_data.get("error"), - "time_cost": tool_meta_data.get("time_cost", 0), - "tool_name": tool_name, - "tool_label": tool_label, - "tool_input": tool_input, - "tool_output": tool_output, - "tool_parameters": tool_meta_data.get("tool_parameters", {}), - "tool_icon": tool_icon, - } - ) - - result["iterations"].append( - { - "tokens": agent_thought.tokens, - "tool_calls": tool_calls, - "tool_raw": { - "inputs": agent_thought.tool_input, - "outputs": agent_thought.observation, - }, - "thought": agent_thought.thought, - "created_at": agent_thought.created_at.isoformat(), - "files": agent_thought.files, - } - ) - - return result - @classmethod def list_agent_providers(cls, user_id: str, tenant_id: str) -> list[PluginAgentProviderEntity]: """ diff --git a/api/services/app/console_gateway.py b/api/services/app/console_gateway.py index 570d3b3be8b99a..8835b6fd1fca6f 100644 --- a/api/services/app/console_gateway.py +++ b/api/services/app/console_gateway.py @@ -6,10 +6,9 @@ from collections.abc import Callable, Sequence from contextlib import AbstractContextManager from dataclasses import dataclass, replace -from typing import Any, BinaryIO, Final, Literal, override +from typing import TYPE_CHECKING, Any, BinaryIO, Final, Literal, override from uuid import UUID, uuid4 -from sqlalchemy import select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, sessionmaker @@ -20,16 +19,16 @@ from graphon.enums import BuiltinNodeTypes from graphon.model_runtime.entities.llm_entities import LLMMode from graphon.model_runtime.entities.message_entities import PromptMessageRole -from libs.datetime_utils import naive_utc_now from machinery.context import RequestContext from models.account import Account from models.agent import AgentIconType from models.model import App, AppMode -from models.tools import WorkflowToolProvider +from models.tool_runtime_contracts import WorkflowToolQueries, WorkflowToolStore +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository from repositories.app.console_repository import ConsoleAppRepository, console_app_actor, require_console_app from repositories.app.response import app_record +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.agent.errors import AgentNameConflictError -from services.agent.home_snapshot_service import AgentHomeSnapshotService from services.agent.roster_package_entities import RosterAgentPackageExport from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.agent.roster_package_importer import RosterAgentPackageImporter @@ -76,11 +75,15 @@ from services.rbac.members import MemberService from services.recommended_app_package_service import RecommendedAppPackageService from services.system_feature_service import SystemFeatureService +from services.tools.provider_queries import ToolProviders from services.tools.workflow_tools_manage_service import WorkflowToolManageService -from services.workflow_service import WorkflowService +from services.workflow.contracts import PreparedDraftSync from services.workflow_tool_references import WorkflowToolTarget from tasks.initialize_created_app_rbac_access_task import initialize_created_app_rbac_access_task +if TYPE_CHECKING: + from services.data_migration.import_service import WorkflowPublisher + # Positions as the console editor lays out its templates (START_INITIAL_POSITION and # NODE_WIDTH_X_OFFSET in web/app/components/workflow/constants.ts). _START_POSITION: Final = (80, 282) @@ -269,6 +272,8 @@ def __init__( agent_packages: RosterAgentPackageExporter, agent_importer: RosterAgentPackageImporter, recommended_packages: RecommendedAppPackageService, + workflow_tools: WorkflowToolStore, + workflow_publisher: WorkflowPublisher, ) -> None: self._session_factory = session_factory self._dsl_factory = dsl_factory @@ -276,6 +281,9 @@ def __init__( self._agent_packages = agent_packages self._agent_importer = agent_importer self._recommended_packages = recommended_packages + self._workflow_tool_queries = workflow_tools + self._workflow_tools = WorkflowToolManageService(workflow_tools) + self._workflow_publisher = workflow_publisher @override def download_import(self, url: str) -> AbstractContextManager[BinaryIO]: @@ -424,19 +432,11 @@ def plan_workflow_tools( workspace_id = context.active_workspace_id with self._session_factory() as session: account = console_app_actor(session, context) - existing = { - provider.id: provider.name - for provider in session.scalars( - select(WorkflowToolProvider).where( - WorkflowToolProvider.tenant_id == workspace_id, - WorkflowToolProvider.id.in_([tool.provider_id for tool in tools]), - ) - ) - } - taken_names = set( - session.scalars(select(WorkflowToolProvider.name).where(WorkflowToolProvider.tenant_id == workspace_id)) - ) can_manage_tools = account.is_admin_or_owner + providers = self._workflow_tool_queries.providers(tenant_id=workspace_id) + requested_ids = {tool.provider_id for tool in tools} + existing = {provider.id: provider.name for provider in providers if provider.id in requested_ids} + taken_names = {provider.name for provider in providers} missing = [tool for tool in tools if tool.provider_id not in existing] if missing and dify_config.RBAC_ENABLED: can_manage_tools = rbac_service.RBACService.CheckAccess.check( @@ -462,14 +462,8 @@ def create_workflow_tool( self, context: RequestContext, *, app_id: str, tool: PackagedWorkflowTool, target: WorkflowToolTarget ) -> None: """Publish an imported workflow and expose it as the Workflow Tool its callers already reference.""" - with self._session_factory.begin() as session: - account = console_app_actor(session, context) - app = require_console_app(session, context, app_id) - workflow = WorkflowService().publish_workflow(session=session, app_model=app, account=account) - app.workflow_id = workflow.id - app.updated_by = account.id - app.updated_at = naive_utc_now() - WorkflowToolManageService.create_workflow_tool( + self._workflow_publisher.publish(context, app_id, marked_name="", marked_comment="") + self._workflow_tools.create_workflow_tool( user_id=context.account_id, tenant_id=context.active_workspace_id, workflow_app_id=app_id, @@ -540,8 +534,15 @@ class AppLifecycleGateway(AppLifecycle): the App transaction; notification and resource cleanup run after it closes. """ - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, + ) -> None: self._session_factory = session_factory + self._responses = AppResponseGateway(tool_providers=tool_providers, workflow_queries=workflow_queries) @override def create(self, context: RequestContext, params: CreateAppParams, settings: AppCreationSettings) -> AppRecord: @@ -571,15 +572,14 @@ def create(self, context: RequestContext, params: CreateAppParams, settings: App @override def create_draft(self, context: RequestContext, app_id: str) -> None: - with self._session_factory() as session: + with self._session_factory.begin() as session: app = require_console_app(session, context, app_id) - WorkflowService(self._session_factory).sync_draft_workflow( + WorkflowDefinitionStore.sync_draft_workflow( app_model=app, graph=copy.deepcopy(_STARTER_DRAFT_GRAPHS[AppMode(app.mode)]), features=copy.deepcopy(_STARTER_DRAFT_FEATURES), - unique_hash=None, - account=console_app_actor(session, context), - environment_variables=[], + account_id=context.account_id, + prepared=PreparedDraftSync(source=None, environment=None), conversation_variables=[], session=session, ) @@ -594,8 +594,8 @@ def delete(self, context: RequestContext, app_id: str) -> AppDeletion: session=session, tenant_id=context.active_workspace_id, binding_id=binding_id ) snapshots = ( - AgentHomeSnapshotService.retire_all_for_agent( - session=session, tenant_id=context.active_workspace_id, agent_id=deleted.backing_agent_id + AgentHomeSnapshotRepository(session).retire_all_for_agent( + tenant_id=context.active_workspace_id, agent_id=deleted.backing_agent_id ) if deleted.backing_agent_id is not None else [] @@ -623,11 +623,11 @@ def updated(self, context: RequestContext, app: AppRecord) -> None: @override def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: - AppService.notify_deleted_app(deleted, account_id=context.account_id) + AppService.notify_deleted_app(deleted, account_id=context.account_id, sessions=self._session_factory) @override def present(self, context: RequestContext, app: AppRecord, *, mask_credentials: bool = False) -> AppRecord: app = replace( app, deleted_tools=AppResponseGateway.find_deleted_tools(context.active_workspace_id, app.tool_references) ) - return AppResponseGateway.mask_record(context, app) if mask_credentials else app + return self._responses.mask_record(context, app) if mask_credentials else app diff --git a/api/services/app/generation/adapters/agent_app.py b/api/services/app/generation/adapters/agent_app.py index 97c66a73fc186d..8799085e098c9a 100644 --- a/api/services/app/generation/adapters/agent_app.py +++ b/api/services/app/generation/adapters/agent_app.py @@ -316,6 +316,17 @@ def _generate_worker( except GenerateTaskStoppedError: pass except Exception as e: + from core.ops.base_trace_instance import application_tracing_hooks + + tracing_hooks = application_tracing_hooks() + if tracing_hooks is not None: + tracing_hooks.capture_agent_app_terminal( + tenant_id=application_generate_entity.app_config.tenant_id, + app_id=application_generate_entity.app_config.app_id, + message_id=message_id, + outcome="failed", + output=None, + ) logger.exception("Unknown Error in Agent App generate worker") queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER) diff --git a/api/services/app/generation/adapters/agent_chat.py b/api/services/app/generation/adapters/agent_chat.py index c000ee5edb94f4..fcb7a96957d856 100644 --- a/api/services/app/generation/adapters/agent_chat.py +++ b/api/services/app/generation/adapters/agent_chat.py @@ -7,7 +7,7 @@ from flask import Flask, current_app from pydantic import ValidationError -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from configs import dify_config from constants import UUID_NIL @@ -20,75 +20,30 @@ from core.app.apps.exc import GenerateTaskStoppedError from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, InvokeFrom -from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.helper.trace_id_helper import extract_trace_session_id_from_args from core.ops.ops_trace_manager import TraceQueueManager -from core.tools.__base.tool import Tool -from core.tools.entities.tool_entities import ToolInvokeMeta from factories import file_factory from graphon.model_runtime.errors.invoke import InvokeAuthorizationError from libs.flask_utils import preserve_flask_contexts from models import Account, App, EndUser from models.annotation_reply import AnnotationReplies -from models.model import Message from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.adapters.agent_chat_runner import AgentChatAppRunner from services.app.generation.message_records import MessageBasedAppGenerator -from services.app.generation.ports import ChatRecords, MessageFileWriter -from services.tools.tool_engine import ToolEngine +from services.app.generation.ports import ChatRecords from services.workflow.execution.ports import WorkflowRuntime logger = logging.getLogger(__name__) -class _SessionBoundAgentToolInvoker(AgentToolInvoker): - """Own one short-lived Session for each eager Agent tool invocation.""" - - def __init__(self, tool_sessions: sessionmaker[Session]) -> None: - self._tool_sessions = tool_sessions - - def __call__( - self, - tool: Tool, - tool_parameters: str | dict[str, Any], - user_id: str, - tenant_id: str, - message: Message, - invoke_from: InvokeFrom, - agent_tool_callback: DifyAgentCallbackHandler, - trace_manager: TraceQueueManager | None = None, - conversation_id: str | None = None, - app_id: str | None = None, - message_id: str | None = None, - *, - records: MessageFileWriter, - ) -> tuple[str, list[str], ToolInvokeMeta]: - with self._tool_sessions() as session: - return ToolEngine.agent_invoke( - session=session, - tool=tool, - tool_parameters=tool_parameters, - user_id=user_id, - tenant_id=tenant_id, - message=message, - invoke_from=invoke_from, - agent_tool_callback=agent_tool_callback, - trace_manager=trace_manager, - conversation_id=conversation_id, - app_id=app_id, - message_id=message_id, - records=records, - ) - - class AgentChatAppGenerator(MessageBasedAppGenerator): def __init__( self, *, dataset_tools: AgentDatasetTools, + tool_invoker: AgentToolInvoker, records: ChatRecords, annotations: AnnotationReplies, - tool_sessions: sessionmaker[Session], draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None = None, workflow_runtime: WorkflowRuntime | None = None, ): @@ -99,7 +54,7 @@ def __init__( workflow_runtime=workflow_runtime, ) self._dataset_tools = dataset_tools - self._tool_invoker = _SessionBoundAgentToolInvoker(tool_sessions) + self._tool_invoker = tool_invoker @overload def generate( diff --git a/api/services/app/generation/adapters/message_pipeline.py b/api/services/app/generation/adapters/message_pipeline.py index 7d4fdbb4eef8ac..bb29023b12b722 100644 --- a/api/services/app/generation/adapters/message_pipeline.py +++ b/api/services/app/generation/adapters/message_pipeline.py @@ -324,9 +324,19 @@ def _process_stream_response( # A cancelled Agent can report authoritative usage after the SSE # stream stops. The repository preserves it atomically with the answer. - self._save_message( + saved_message = self._save_message( trace_manager=trace_manager, preserve_existing_usage=isinstance(event, QueueStopEvent) ) + if ( + isinstance(event, QueueStopEvent) + and self._application_generate_entity.app_config.app_mode == AppMode.AGENT + ): + # Provider/billing subscribers may perform external I/O. + # The repository has released the stop-save row lock before this notification. + message_was_created.send( + saved_message, + application_generate_entity=self._application_generate_entity, + ) message_end_resp = self._message_end_to_stream_response() yield message_end_resp case QueueRetrieverResourcesEvent(): @@ -423,11 +433,19 @@ def _save_message( *, trace_manager: TraceQueueManager | None = None, preserve_existing_usage: bool = False, - ): + ) -> Message: llm_result = self._task_state.llm_result usage = llm_result.usage usage.latency = time.perf_counter() - self.start_at self._task_state.metadata.usage = usage + agent_stop_requested = ( + preserve_existing_usage and self._application_generate_entity.app_config.app_mode == AppMode.AGENT + ) + metadata = self._task_state.metadata.model_dump(mode="json") + if agent_stop_requested: + # Closing the consumer records a request, not a runtime outcome. + # The Agent worker owns the terminal after cancellation/cleanup. + metadata["agent_stop_requested"] = True saved = self._records.save_message( self._identity, MessageUpdate( @@ -437,7 +455,7 @@ def _save_message( else "" ), latency=usage.latency, - metadata=self._task_state.metadata.model_dump(mode="json"), + metadata=metadata, files=[], usage=usage, prompt=PromptMessageUtil.prompt_messages_to_prompt_for_saving( @@ -449,7 +467,7 @@ def _save_message( metadata = json.loads(saved.message_metadata) if saved.message_metadata else {} if "usage" in metadata: self._task_state.llm_result.usage = LLMUsage.from_metadata(metadata["usage"]) - if trace_manager: + if trace_manager and not agent_stop_requested: trace_manager.add_trace_task( TraceTask( TraceTaskName.MESSAGE_TRACE, @@ -459,7 +477,9 @@ def _save_message( ) ) # Consumers observe a committed, detached message. - message_was_created.send(saved, application_generate_entity=self._application_generate_entity) + if not agent_stop_requested: + message_was_created.send(saved, application_generate_entity=self._application_generate_entity) + return saved def _handle_stop(self, event: QueueStopEvent): """ diff --git a/api/services/app/generation/errors.py b/api/services/app/generation/errors.py index bdcd38abfdffa4..51b4e77116a291 100644 --- a/api/services/app/generation/errors.py +++ b/api/services/app/generation/errors.py @@ -1,24 +1,6 @@ -from core.app.apps.exc import AppGenerateError - -AGENT_SESSION_CONFIGURATION_CHANGED_ERROR_CODE = "agent_session_configuration_changed" -AGENT_SESSION_CONFIGURATION_CHANGED_MESSAGE = ( - "The Agent configuration changed after this conversation started. Start a new conversation to continue." -) - - class AgentAppGeneratorError(ValueError): """Raised when an Agent App turn cannot be set up.""" class AgentAppNotPublishedError(AgentAppGeneratorError): """Raised when a public Agent App runtime is requested before publish.""" - - -class AgentSessionSnapshotIncompatibleError(AppGenerateError): - """Raised when a retained session snapshot no longer matches the current composition.""" - - error_code = AGENT_SESSION_CONFIGURATION_CHANGED_ERROR_CODE - status_code = 409 - - def __init__(self) -> None: - super().__init__(AGENT_SESSION_CONFIGURATION_CHANGED_MESSAGE) diff --git a/api/services/app/generation/input_adapter.py b/api/services/app/generation/input_adapter.py index a103028758edd2..0239548f749265 100644 --- a/api/services/app/generation/input_adapter.py +++ b/api/services/app/generation/input_adapter.py @@ -1,8 +1,7 @@ import logging from collections.abc import Callable, Mapping, Sequence from contextlib import AbstractContextManager, nullcontext -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, ClassVar, final +from typing import TYPE_CHECKING, Any, final from core.app.apps.draft_variable_saver import ( DraftVariableSaver, @@ -28,13 +27,20 @@ logger = logging.getLogger(__name__) -@dataclass(kw_only=True) -class AppInputAdapter: - _file_access_controller: ClassVar[DatabaseFileAccessController] = DatabaseFileAccessController() +# This behavior-rich base is subclassed by generators; dataclass equality and repr would alter their semantics. +class AppInputAdapter: # noqa: B903 + _file_access_controller: DatabaseFileAccessController = DatabaseFileAccessController() - draft_variable_loader: Callable[[WorkflowSnapshot, str], VariableLoader] | None = None - draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None = None - workflow_runtime: WorkflowRuntime | None = None + def __init__( + self, + *, + draft_variable_loader: Callable[[WorkflowSnapshot, str], VariableLoader] | None = None, + draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None = None, + workflow_runtime: WorkflowRuntime | None = None, + ) -> None: + self.draft_variable_loader = draft_variable_loader + self.draft_variable_saver = draft_variable_saver + self.workflow_runtime = workflow_runtime @staticmethod def _bind_file_access_scope( diff --git a/api/services/app/generation/runtime.py b/api/services/app/generation/runtime.py index 36761ab096f390..2233a022cbb59f 100644 --- a/api/services/app/generation/runtime.py +++ b/api/services/app/generation/runtime.py @@ -2,7 +2,7 @@ from typing import Protocol -from services.agent.chat.ports import AgentDatasetTools +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker from services.app.generation.agent_config import AgentAppConfigurations from services.workflow.execution.chatflow_ports import ChatflowRuntime @@ -12,3 +12,5 @@ class AppGenerationRuntime(ChatflowRuntime, Protocol): def dataset_tools(self) -> AgentDatasetTools: ... @property def agent_configs(self) -> AgentAppConfigurations: ... + @property + def agent_tool_invoker(self) -> AgentToolInvoker: ... diff --git a/api/services/app/response_gateway.py b/api/services/app/response_gateway.py index 1eeb3b5a833133..fd57681a4c8201 100644 --- a/api/services/app/response_gateway.py +++ b/api/services/app/response_gateway.py @@ -7,16 +7,22 @@ from pydantic import JsonValue -from core.agent.tool_configuration import mask_agent_tool_parameters from core.plugin.plugin_service import PluginService -from core.tools.tool_manager import ToolManager from machinery.context import RequestContext from models.model import AppMode from models.provider_ids import GenericProviderID +from models.tool_runtime_contracts import WorkflowToolQueries from services.entities.app_entities import AppRecord, AppToolReference +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.provider_queries import ToolProviders +from services.tools.tool_manager import ToolManager class AppResponseGateway: + def __init__(self, tool_providers: ToolProviders, workflow_queries: WorkflowToolQueries) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries + @staticmethod def find_deleted_tools(tenant_id: str, references: tuple[AppToolReference, ...]) -> list[dict[str, str]]: """Resolve builtin tool availability from detached references, without a database session.""" @@ -47,12 +53,13 @@ def find_deleted_tools(tenant_id: str, references: tuple[AppToolReference, ...]) or (ref.provider_type == "builtin" and existence.get(ref.provider_id) is False) ] - @staticmethod - def mask_record(context: RequestContext, app: AppRecord) -> AppRecord: + def mask_record(self, context: RequestContext, app: AppRecord) -> AppRecord: if app.mode_compatible_with_agent != AppMode.AGENT_CHAT or app.app_model_config is None: return app config = deepcopy(app.app_model_config) config["agent_mode"] = mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=cast(Mapping[str, JsonValue], config.get("agent_mode", {})), tenant_id=context.active_workspace_id, app_id=app.id, diff --git a/api/services/app_definition_query_service.py b/api/services/app_definition_query_service.py index 4b3731e7f011a3..0f1513fb285fc0 100644 --- a/api/services/app_definition_query_service.py +++ b/api/services/app_definition_query_service.py @@ -5,7 +5,7 @@ from typing import Any, NamedTuple, Protocol from core.app.app_config.common.parameters_mapping import AppParametersDict, get_parameters_from_feature_dict -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.errors.app import AppAbnormalStatusError, AppApiDisabledError from services.errors.workspace import WorkspaceArchivedError, WorkspaceNotFoundError diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index e8adcef384913d..0ecded91808f11 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -51,22 +51,25 @@ from graphon.nodes.tool.entities import ToolNodeData, ToolProviderType from libs.datetime_utils import naive_utc_now from libs.url_utils import github_blob_to_raw_url +from machinery.context import RequestContext from models import Account, App, AppMode from models.agent import AgentScope, WorkflowAgentBindingType, WorkflowAgentNodeBinding from models.agent_config_entities import WorkflowNodeJobConfig from models.model import AppModelConfig, AppModelConfigDict, IconType, load_annotation_reply_config from models.workflow import Workflow +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.agent.dsl_entities import AGENT_NODE_JOB_DSL_KEY, AgentPackage, make_agent_app_dsl -from services.agent.dsl_service import AgentDslService, is_agent_v2_graph -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.agent.dsl_service import AgentDslService from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.agent.package_resource_exporter import AgentPackageResourceExporter +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.dsl_content import DSL_MAX_SIZE, dsl_content_size from services.dsl_version import check_version_compatibility from services.enterprise import rbac_service from services.enterprise.rbac_service import RBACService from services.entities.dsl_entities import ( AppDslExportData, + AppDslOverwriteStore, AppImportPackage, CheckDependenciesResult, DslImportWarning, @@ -87,9 +90,16 @@ ) from services.knowledge_fs.knowledge_retrieval_v2.validation import missing_control_space_ids from services.plugin.dependencies_analysis import DependenciesAnalysisService +from services.workflow.contracts import ( + DraftImportResult, + DraftSyncCommand, + PreparedEnvironmentVariables, + WorkflowOwner, + WorkflowSnapshot, +) +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.environment_variable_service import prepare_environment_variables from services.workflow.graph_diff import draft_token -from services.workflow_draft_variable_service import WorkflowDraftVariableService -from services.workflow_service import WorkflowService logger = logging.getLogger(__name__) @@ -159,8 +169,16 @@ class CheckDependenciesPendingData(BaseModel): class AppDslService: _warnings: list[DslImportWarning] - def __init__(self, session: Session): + def __init__( + self, + session: Session, + *, + drafts: WorkflowDraftService[WorkflowAgentBindingStore], + overwrites: AppDslOverwriteStore, + ): self._session = session + self._drafts = drafts + self._overwrites = overwrites self._warnings = [] @staticmethod @@ -321,17 +339,24 @@ def import_app( # Authorize overwrites in a separate read session before storage I/O. # The normal import below reloads and rechecks the target after upload. if app_id: - with Session(self._session.get_bind()) as authorization_session: - target = AppDslService(authorization_session)._load_app_for_overwrite(account, app_id) - if target is None: - raise ValueError("App not found") - self._validate_workflow_overwrite(target, data) + target = self._overwrites.snapshot( + tenant_id=tenant_id, + account_id=account.id, + app_id=app_id, + rbac_allowed=self._check_overwrite_access(account, app_id), + ) + if target is None: + raise ValueError("App not found") + self._validate_workflow_overwrite(target.mode, data) package.materialize_icons(data=data, tenant_id=tenant_id, account_id=account.id) agents, self._warnings = package.materialize_agents(tenant_id=tenant_id, account_id=account.id) if agents: data["agent_packages"] = agents content = yaml.safe_dump(data, allow_unicode=True) + environment = ( + self._prepare_import_environment(data, account, app_id) if status != ImportStatus.PENDING else None + ) # If app_id is provided, check if it exists app = None if app_id: @@ -350,7 +375,7 @@ def import_app( error="Only workflow or advanced chat apps can be overwritten", ) - self._validate_workflow_overwrite(app, data) + self._validate_workflow_overwrite(AppMode(app.mode), data) # If major version mismatch, store import info in Redis if status == ImportStatus.PENDING: @@ -415,12 +440,11 @@ def import_app( dependencies=check_dependencies_pending_data, import_app_id=import_app_id, allow_premium_site_settings=allow_premium_site_settings, + environment=environment, preserve_agent_bindings=preserve_agent_bindings, draft_hash=draft_hash, ) - draft_var_srv = WorkflowDraftVariableService(session=self._session) - draft_var_srv.delete_app_workflow_variables(app_id=app.id) result_status = self._status_with_warnings(status) return Import( id=import_id, @@ -495,6 +519,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: raise ValueError("Current tenant is not set") allow_premium_site_settings = FeatureService.can_import_premium_site_settings(tenant_id) + environment = self._prepare_import_environment(data, account, pending_data.app_id) app = None if pending_data.app_id: app = self._load_app_for_overwrite(account, pending_data.app_id) @@ -506,7 +531,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: ) if app is not None: - self._validate_workflow_overwrite(app, data) + self._validate_workflow_overwrite(AppMode(app.mode), data) # Create or update app app_created = app is None @@ -520,6 +545,7 @@ def confirm_import(self, *, import_id: str, account: Account) -> Import: icon=pending_data.icon, icon_background=pending_data.icon_background, allow_premium_site_settings=allow_premium_site_settings, + environment=environment, draft_hash=pending_data.draft_hash, preserve_agent_bindings=pending_data.preserve_agent_bindings, ) @@ -584,30 +610,25 @@ def cache_import_dependencies(*, app_id: str, dependencies: list[PluginDependenc CheckDependenciesPendingData(app_id=app_id, dependencies=dependencies).model_dump_json(), ) - def _load_app_for_overwrite(self, account: Account, app_id: str) -> App | None: + def _check_overwrite_access(self, account: Account, app_id: str) -> bool: if account.current_tenant_id is None: raise ValueError("Current tenant is not set") if dify_config.RBAC_ENABLED and self._session.in_transaction(): raise RuntimeError("App overwrite authorization requires a session without an active transaction") - rbac_allowed = not dify_config.RBAC_ENABLED or RBACService.CheckAccess.check( + return not dify_config.RBAC_ENABLED or RBACService.CheckAccess.check( account.current_tenant_id, account.id, scene=RBACPermission.APP_IMPORT_EXPORT_DSL, resource_type=RBACResourceScope.APP, resource_id=app_id, ) - app = self._session.scalar( - select(App) - .where( - App.id == app_id, - App.tenant_id == account.current_tenant_id, - App.status == "normal", - ) - .execution_options(populate_existing=True) + + def _load_app_for_overwrite(self, account: Account, app_id: str) -> App | None: + allowed = self._check_overwrite_access(account, app_id) + assert account.current_tenant_id is not None + return self._overwrites.load( + tenant_id=account.current_tenant_id, account_id=account.id, app_id=app_id, rbac_allowed=allowed ) - if app is not None and not rbac_allowed and app.maintainer != account.id: - raise NoPermissionError("You do not have permission to overwrite this app") - return app def _ensure_agent_import_permission(self, account: Account, *, app: App | None) -> None: if not dify_config.RBAC_ENABLED: @@ -632,10 +653,10 @@ def _ensure_agent_import_permission(self, account: Account, *, app: App | None) raise NoPermissionError("Agent DSL import permission is required to import an Agent App") @staticmethod - def _validate_workflow_overwrite(app: App, data: dict[str, Any]) -> None: + def _validate_workflow_overwrite(target_mode: AppMode, data: dict[str, Any]) -> None: """Apply editor compatibility checks to both YAML and package imports.""" app_mode = data.get("app", {}).get("mode") - if app.mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT} or app_mode not in { + if target_mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT} or app_mode not in { AppMode.WORKFLOW, AppMode.ADVANCED_CHAT, }: @@ -643,13 +664,46 @@ def _validate_workflow_overwrite(app: App, data: dict[str, Any]) -> None: # Package uploads cannot run the editor's YAML node checks before import. invalid_types = ( {BuiltinNodeTypes.END, "trigger-webhook", "trigger-schedule", "trigger-plugin"} - if app.mode == AppMode.ADVANCED_CHAT + if target_mode == AppMode.ADVANCED_CHAT else {BuiltinNodeTypes.ANSWER} ) nodes = data.get("workflow", {}).get("graph", {}).get("nodes", []) if any(node.get("data", {}).get("type") in invalid_types for node in nodes): raise ValueError("Workflow contains node types incompatible with the target App") + def _prepare_import_environment( + self, data: dict[str, Any], account: Account, app_id: str | None + ) -> PreparedEnvironmentVariables | None: + """Prepare secrets before the import starts writing App, Workflow and Agent rows.""" + if data.get("app", {}).get("mode") not in (AppMode.WORKFLOW, AppMode.ADVANCED_CHAT): + return None + workflow_data = data.get("workflow") + if not isinstance(workflow_data, dict): + return None + tenant_id = account.current_tenant_id + if tenant_id is None: + raise ValueError("Current tenant is not set") + source = None + if app_id is not None: + target = self._overwrites.snapshot( + tenant_id=tenant_id, + account_id=account.id, + app_id=app_id, + rbac_allowed=self._check_overwrite_access(account, app_id), + ) + if target is None: + raise ValueError("App not found") + self._validate_workflow_overwrite(target.mode, data) + source = target.workflow + return prepare_environment_variables( + tenant_id=tenant_id, + source=source, + variables=[ + variable_factory.build_environment_variable_from_mapping(value) + for value in workflow_data.get("environment_variables", []) + ], + ) + def _lock_draft_matching(self, *, app: App | None, draft_hash: str) -> Workflow: """Lock the draft the caller exported, or raise DraftChangedError when it has moved on. @@ -657,7 +711,7 @@ def _lock_draft_matching(self, *, app: App | None, draft_hash: str) -> Workflow: """ if app is None: raise DraftChangedError() - draft = WorkflowService().get_draft_workflow_for_update(app_model=app, session=self._session) + draft = WorkflowDefinitionStore.get_draft_workflow(app, session=self._session, for_update=True) if draft is None or draft_token(draft) != draft_hash: raise DraftChangedError() return draft @@ -676,6 +730,7 @@ def _create_or_update_app( dependencies: list[PluginDependency] | None = None, import_app_id: str | None = None, allow_premium_site_settings: bool = True, + environment: PreparedEnvironmentVariables | None = None, preserve_agent_bindings: bool = False, draft_hash: str | None = None, ) -> App: @@ -686,7 +741,8 @@ def _create_or_update_app( raise ValueError("loss app mode") app_mode = AppMode(app_mode) # Check the draft before any write so a refused import leaves the app and its cached dependencies alone. - locked_draft = self._lock_draft_matching(app=app, draft_hash=draft_hash) if draft_hash is not None else None + if draft_hash is not None: + self._lock_draft_matching(app=app, draft_hash=draft_hash) site_data = SiteDsl.model_validate(data["site"]) if data.get("site") is not None else None if app_mode == AppMode.AGENT: self._ensure_agent_import_permission(account, app=app) @@ -766,21 +822,6 @@ def _create_or_update_app( if not workflow_data or not isinstance(workflow_data, dict): raise ValueError("Missing workflow data for workflow/advanced chat app") - environment_variables_list = workflow_data.get("environment_variables", []) - environment_variables = [ - variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list - ] - conversation_variables_list = workflow_data.get("conversation_variables", []) - conversation_variables = [ - variable_factory.build_conversation_variable_from_mapping(obj) - for obj in conversation_variables_list - ] - - workflow_service = WorkflowService() - current_draft_workflow = locked_draft or workflow_service.get_draft_workflow( - app_model=app, session=self._session - ) - unique_hash = current_draft_workflow.unique_hash if current_draft_workflow else None graph = workflow_data.get("graph", {}) if not isinstance(graph, dict): raise ValueError("Workflow graph must be a mapping") @@ -819,46 +860,54 @@ def _create_or_update_app( raw_agent_packages = data.get("agent_packages") or {} if not isinstance(raw_agent_packages, Mapping): raise ValueError("agent_packages must be a mapping") - import_agent_bindings = bool(raw_agent_packages) or ( - preserve_agent_bindings and is_agent_v2_graph(graph) + agent_packages = dict(raw_agent_packages) + import_agent_bindings = bool(agent_packages) or ( + preserve_agent_bindings and any(WorkflowAgentNodeValidator.iter_agent_v2_nodes(graph)) ) - graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if raw_agent_packages else graph - draft_workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph_for_sync, - features=workflow_data.get("features", {}), - unique_hash=unique_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=self._session, - commit=not import_agent_bindings, - sync_agent_bindings=not import_agent_bindings, - ) - if import_agent_bindings: - retirement_candidates: set[str] = set() - if raw_agent_packages: - _, warnings, retirement_candidates = AgentDslService(self._session).import_workflow_packages( - workflow=draft_workflow, + graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if agent_packages else graph + + def materialize(workflow: WorkflowSnapshot) -> DraftImportResult: + imported_graph = graph + retired: set[str] = set() + if agent_packages: + imported_graph, warnings, retired = AgentDslService(self._session).import_workflow_packages( + workflow=workflow, portable_graph=graph, - raw_packages=raw_agent_packages, + raw_packages=agent_packages, account=account, ) self._warnings.extend(warnings) if preserve_agent_bindings: - retirement_candidates.update( - self._sync_copied_workflow_bindings(workflow=draft_workflow, account_id=account.id) + workflow_row = self._session.get(Workflow, workflow.id) + if workflow_row is None: + raise WorkflowNotFoundError("Missing draft workflow configuration, please check.") + workflow_row.graph = json.dumps(imported_graph) + self._session.flush() + retired.update( + self._sync_copied_workflow_bindings(workflow=workflow_row, account_id=account.id) ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) - self._session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) + imported_graph = workflow_row.graph_dict + return DraftImportResult(dict(imported_graph), retired) + + self._drafts.sync( + RequestContext(str(uuid4()), None, account.id, app.tenant_id), + WorkflowOwner(app.id), + DraftSyncCommand( + graph=graph_for_sync, + features=workflow_data.get("features", {}), + unique_hash=None, + # DSL replaces the entire definition under the write lock. + check_hash=False, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=workflow_data.get("environment_variables", []), + conversation_variables=workflow_data.get("conversation_variables", []), + clear_debug_variables=True, + ), + materialize=materialize if import_agent_bindings else None, + environment=environment, + ) case AppMode.CHAT | AppMode.AGENT_CHAT | AppMode.COMPLETION: # Initialize model config model_config = data.get("model_config") @@ -1028,11 +1077,11 @@ def serialize_export_data(prepared: AppDslExportData) -> str: def _load_export_workflow(app_model: App, *, workflow_id: str | None, session: Session) -> Workflow: """Require the selected published version, or the app draft when no ID is supplied.""" if workflow_id: - workflow = WorkflowService.get_published_workflow_by_id(app_model, workflow_id, session=session) + workflow = WorkflowDefinitionStore.get_published_workflow_by_id(app_model, workflow_id, session=session) if workflow is None: raise WorkflowNotFoundError(f"Workflow version not found. Workflow ID: {workflow_id}.") else: - workflow = WorkflowService.get_draft_workflow(app_model, session=session) + workflow = WorkflowDefinitionStore.get_draft_workflow(app_model, session=session) if workflow is None: raise WorkflowNotFoundError("Missing draft workflow configuration, please check.") return workflow diff --git a/api/services/app_generate_service.py b/api/services/app_generate_service.py index c3881dcce80d33..7fe482f14bdce4 100644 --- a/api/services/app_generate_service.py +++ b/api/services/app_generate_service.py @@ -4,37 +4,39 @@ import threading import uuid from collections.abc import Callable, Mapping -from typing import TYPE_CHECKING, Any +from typing import Any from sqlalchemy.orm import Session from configs import dify_config -from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator -from core.app.apps.agent_app.app_generator import AgentAppGenerator -from core.app.apps.agent_chat.app_generator import AgentChatAppGenerator -from core.app.apps.chat.app_generator import ChatAppGenerator -from core.app.apps.completion.app_generator import CompletionAppGenerator -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.features.rate_limiting import RateLimit from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig from core.db import session_factory +from core.trigger.constants import is_trigger_node_type from enums import DeploymentEdition, QuotaType from extensions.otel import AppGenerateHandler, trace_span from models.model import Account, App, AppMode, EndUser -from models.workflow import Workflow +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.app.generation.adapters.agent_app import AgentAppGenerator +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.adapters.completion import CompletionAppGenerator +from services.app.generation.response import convert_to_event_stream +from services.app.generation.runtime import AppGenerationRuntime from services.errors.app import ( QuotaExceededError, TriggerWorkflowServiceModeUnavailableError, - WorkflowIdFormatError, - WorkflowNotFoundError, ) from services.errors.llm import InvokeRateLimitError from services.quota_service import QuotaService, unlimited +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.response_stream import WorkflowEventStream +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.stream_reservation import WorkflowStreamReservation +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_execution_identity import build_workflow_execution_identity -from services.workflow_run_agg import WorkflowRunAgg -from services.workflow_service import WorkflowService from tasks.app_generate.workflow_execute_task import AppExecutionParams, workflow_based_app_execution_task logger = logging.getLogger(__name__) @@ -48,9 +50,6 @@ } ) -if TYPE_CHECKING: - from controllers.console.app.workflow import LoopNodeRunPayload - class AppGenerateService: @staticmethod @@ -106,8 +105,11 @@ def generate( invoke_from: InvokeFrom, *, session: Session, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, root_node_id: str | None = None, + workflow_snapshot: WorkflowSnapshot | None = None, ): """ App Content Generate @@ -116,8 +118,10 @@ def generate( :param args: args :param invoke_from: invoke from :param streaming: streaming + :param workflow_snapshot: trusted trigger-debug revision; None selects the requested/current workflow :return: """ + cls._validate_workflow_snapshot(app_model, invoke_from, workflow_snapshot) return cls._run_with_guardrails( app_model=app_model, streaming=streaming, @@ -128,12 +132,121 @@ def generate( invoke_from=invoke_from, streaming=streaming, root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, session=session, + variables=variables, + runtime=runtime, + rate_limit=rate_limit, + request_id=request_id, + ), + ) + + @staticmethod + def _validate_workflow_snapshot( + app_model: App, invoke_from: InvokeFrom, workflow_snapshot: WorkflowSnapshot | None + ) -> None: + if workflow_snapshot is not None and ( + invoke_from != InvokeFrom.DEBUGGER + or app_model.mode != AppMode.WORKFLOW + or (workflow_snapshot.tenant_id, workflow_snapshot.app_id) != (app_model.tenant_id, app_model.id) + ): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") + + @classmethod + @trace_span(AppGenerateHandler) + def generate_workflow_stream( + cls, + *, + app_model: App, + user: Account | EndUser, + workflow: WorkflowSnapshot, + args: Mapping[str, Any], + invoke_from: InvokeFrom, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, + reservation: WorkflowStreamReservation | None, + ): + """Prepare streaming execution from detached inputs, without a database transaction.""" + if app_model.mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT}: + raise ValueError(f"Invalid app mode {app_model.mode}") + if (workflow.tenant_id, workflow.app_id) != (app_model.tenant_id, app_model.id): + raise ValueError("Workflow does not belong to this app") + cls._validate_workflow_snapshot(app_model, invoke_from, workflow_snapshot) + if app_model.mode == AppMode.WORKFLOW: + cls._ensure_workflow_service_mode_available(workflow=workflow, invoke_from=invoke_from) + return cls._run_with_guardrails( + app_model=app_model, + streaming=True, + action=lambda rate_limit, request_id: cls._stream_workflow( + app_model=app_model, + user=user, + workflow=workflow, + args=args, + invoke_from=invoke_from, + variables=variables, + runtime=runtime, + root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, + reservation=reservation, rate_limit=rate_limit, request_id=request_id, ), ) + @classmethod + def _stream_workflow( + cls, + *, + app_model: App, + user: Account | EndUser, + workflow: WorkflowSnapshot, + args: Mapping[str, Any], + invoke_from: InvokeFrom, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, + rate_limit: RateLimit, + request_id: str, + reservation: WorkflowStreamReservation | None = None, + ): + payload = AppExecutionParams.new( + app_model=app_model, + workflow=workflow, + user=user, + args=args, + invoke_from=invoke_from, + streaming=True, + call_depth=0, + root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, + workflow_run_id=(workflow_snapshot.execution_id if workflow_snapshot is not None else None) + or str(uuid.uuid4()), + ) + payload_json = payload.model_dump_json() + + def on_subscribe(): + workflow_based_app_execution_task.delay(payload_json) + + # Reserved debug executions start only with a subscriber. A fallback timer + # could enqueue after an unopened/failed response releases its Agent pins. + # Let dispatch errors reach the stream owner so it can release the pins. + subscribe = ( + (lambda: reservation.enqueue(on_subscribe)) + if reservation is not None + else cls._build_streaming_task_on_subscribe(on_subscribe) + ) + return rate_limit.generate( + convert_to_event_stream( + WorkflowEventStream.retrieve_events( + AppMode(app_model.mode), payload.workflow_run_id, on_subscribe=subscribe + ) + ), + request_id, + ) + @classmethod def _run_with_guardrails( cls, @@ -178,7 +291,10 @@ def _dispatch_generate( invoke_from: InvokeFrom, streaming: bool, root_node_id: str | None, + workflow_snapshot: WorkflowSnapshot | None, session: Session, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, rate_limit: RateLimit, request_id: str, ): @@ -190,8 +306,12 @@ def _dispatch_generate( match effective_mode: case AppMode.COMPLETION: return rate_limit.generate( - CompletionAppGenerator.convert_to_event_stream( - CompletionAppGenerator().generate( + convert_to_event_stream( + CompletionAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( session=session, app_model=app_model, user=user, @@ -204,8 +324,15 @@ def _dispatch_generate( ) case AppMode.AGENT_CHAT: return rate_limit.generate( - AgentChatAppGenerator.convert_to_event_stream( - AgentChatAppGenerator(execution_driver=WorkflowRunAgg.run).generate( + convert_to_event_stream( + AgentChatAppGenerator( + dataset_tools=runtime.dataset_tools, + tool_invoker=runtime.agent_tool_invoker, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + draft_variable_saver=variables.saver_factory, + workflow_runtime=runtime, + ).generate( session=session, app_model=app_model, user=user, @@ -217,14 +344,21 @@ def _dispatch_generate( request_id, ) case AppMode.AGENT: + session.expunge_all() + session.close() return rate_limit.generate( - AgentAppGenerator.convert_to_event_stream( - AgentAppGenerator().generate( + convert_to_event_stream( + AgentAppGenerator( + agent_configs=runtime.agent_configs, + tool_providers=runtime.tool_providers, + workflow_queries=runtime.tools, + records=runtime.chat_records, + annotations=runtime.annotation_replies, + ).generate( app_model=app_model, user=user, args=args, invoke_from=invoke_from, - session=session, streaming=streaming, ), ), @@ -232,8 +366,12 @@ def _dispatch_generate( ) case AppMode.CHAT: return rate_limit.generate( - ChatAppGenerator.convert_to_event_stream( - ChatAppGenerator().generate( + convert_to_event_stream( + ChatAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( session=session, app_model=app_model, user=user, @@ -249,31 +387,17 @@ def _dispatch_generate( workflow = cls._get_workflow(app_model, invoke_from, workflow_id, session=session) if streaming: - # Streaming mode: subscribe to SSE and enqueue the execution on first subscriber - payload = AppExecutionParams.new( + return cls._stream_workflow( app_model=app_model, - workflow=workflow, user=user, + workflow=workflow, args=args, invoke_from=invoke_from, - streaming=True, - call_depth=0, - workflow_run_id=str(uuid.uuid4()), - ) - payload_json = payload.model_dump_json() - - def on_subscribe(): - workflow_based_app_execution_task.delay(payload_json) - - on_subscribe = cls._build_streaming_task_on_subscribe(on_subscribe) - return rate_limit.generate( - AdvancedChatAppGenerator.convert_to_event_stream( - AdvancedChatAppGenerator.retrieve_events( - AppMode.ADVANCED_CHAT, - payload.workflow_run_id, - on_subscribe=on_subscribe, - ), - ), + variables=variables, + runtime=runtime, + root_node_id=None, + workflow_snapshot=None, + rate_limit=rate_limit, request_id=request_id, ) @@ -283,9 +407,14 @@ def on_subscribe(): session_factory=session_factory.get_session_maker(), state_owner_user_id=workflow.created_by, ) - advanced_generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run) + advanced_generator = AdvancedChatAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ) return rate_limit.generate( - advanced_generator.convert_to_event_stream( + convert_to_event_stream( advanced_generator.generate( app_model=app_model, workflow=workflow, @@ -295,42 +424,29 @@ def on_subscribe(): workflow_run_id=str(uuid.uuid4()), streaming=False, pause_state_config=pause_config, - session=session, ) ), request_id=request_id, ) case AppMode.WORKFLOW: workflow_id = args.get("workflow_id") - workflow = cls._get_workflow(app_model, invoke_from, workflow_id, session=session) + workflow = cls._get_workflow( + app_model, invoke_from, workflow_id, session=session, snapshot=workflow_snapshot + ) cls._ensure_workflow_service_mode_available(workflow=workflow, invoke_from=invoke_from) if streaming: - payload = AppExecutionParams.new( + return cls._stream_workflow( app_model=app_model, - workflow=workflow, user=user, + workflow=workflow, args=args, invoke_from=invoke_from, - streaming=True, - call_depth=0, + variables=variables, + runtime=runtime, root_node_id=root_node_id, - workflow_run_id=str(uuid.uuid4()), - ) - payload_json = payload.model_dump_json() - - def on_subscribe(): - workflow_based_app_execution_task.delay(payload_json) - - on_subscribe = cls._build_streaming_task_on_subscribe(on_subscribe) - return rate_limit.generate( - WorkflowAppGenerator.convert_to_event_stream( - MessageBasedAppGenerator.retrieve_events( - AppMode.WORKFLOW, - payload.workflow_run_id, - on_subscribe=on_subscribe, - ), - ), - request_id, + workflow_snapshot=workflow_snapshot, + rate_limit=rate_limit, + request_id=request_id, ) pause_config = PauseStateLayerConfig( @@ -338,8 +454,13 @@ def on_subscribe(): state_owner_user_id=workflow.created_by, ) return rate_limit.generate( - WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate( + convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).generate( app_model=app_model, workflow=workflow, user=user, @@ -358,9 +479,14 @@ def on_subscribe(): raise ValueError(f"Invalid app mode {app_model.mode}") @staticmethod - def _ensure_workflow_service_mode_available(*, workflow: Workflow, invoke_from: InvokeFrom) -> None: - if invoke_from in _MANUAL_WORKFLOW_INVOKE_SOURCES and workflow.has_trigger_node(): - raise TriggerWorkflowServiceModeUnavailableError() + def _ensure_workflow_service_mode_available(*, workflow: WorkflowSnapshot, invoke_from: InvokeFrom) -> None: + if invoke_from not in _MANUAL_WORKFLOW_INVOKE_SOURCES: + return + + for _, node_data in workflow.walk_nodes(): + node_type = node_data.get("type") + if isinstance(node_type, str) and is_trigger_node_type(node_type): + raise TriggerWorkflowServiceModeUnavailableError() @staticmethod def _get_max_active_requests(app: App) -> int: @@ -391,29 +517,38 @@ def generate_single_iteration( node_id: str, args: Any, *, - session: Session, + workflow: WorkflowSnapshot, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, ): match app_model.mode: case AppMode.COMPLETION | AppMode.CHAT | AppMode.AGENT_CHAT: raise ValueError(f"Invalid app mode {app_model.mode}") case AppMode.ADVANCED_CHAT: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate( + return convert_to_event_stream( + AdvancedChatAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_iteration_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.WORKFLOW: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate( + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_iteration_generate( app_model=app_model, workflow=workflow, node_id=node_id, @@ -421,7 +556,6 @@ def generate_single_iteration( execution_identity=build_workflow_execution_identity(user), args=args, streaming=streaming, - session=session, ) ) case AppMode.CHANNEL | AppMode.RAG_PIPELINE: @@ -435,31 +569,40 @@ def generate_single_loop( app_model: App, user: Account, node_id: str, - args: LoopNodeRunPayload, + args: Mapping[str, Any], *, - session: Session, + workflow: WorkflowSnapshot, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, streaming: bool = True, ): match app_model.mode: case AppMode.COMPLETION | AppMode.CHAT | AppMode.AGENT_CHAT: raise ValueError(f"Invalid app mode {app_model.mode}") case AppMode.ADVANCED_CHAT: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate( + return convert_to_event_stream( + AdvancedChatAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_loop_generate( app_model=app_model, workflow=workflow, node_id=node_id, user=user, args=args, streaming=streaming, - session=session, ) ) case AppMode.WORKFLOW: - workflow = cls._get_workflow(app_model, InvokeFrom.DEBUGGER, session=session) - return AdvancedChatAppGenerator.convert_to_event_stream( - WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate( + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).single_loop_generate( app_model=app_model, workflow=workflow, node_id=node_id, @@ -467,7 +610,6 @@ def generate_single_loop( execution_identity=build_workflow_execution_identity(user), args=args, streaming=streaming, - session=session, ) ) case AppMode.CHANNEL | AppMode.RAG_PIPELINE: @@ -483,7 +625,8 @@ def _get_workflow( workflow_id: str | None = None, *, session: Session, - ) -> Workflow: + snapshot: WorkflowSnapshot | None = None, + ) -> WorkflowSnapshot: """ Get workflow :param app_model: app model @@ -491,32 +634,10 @@ def _get_workflow( :param workflow_id: optional workflow id to specify a specific version :return: """ - workflow_service = WorkflowService() - - # If workflow_id is specified, get the specific workflow version - if workflow_id: - try: - _ = uuid.UUID(workflow_id) - except ValueError as caught_error: - raise WorkflowIdFormatError(f"Invalid workflow_id format: '{workflow_id}'. ") from caught_error - workflow = workflow_service.get_published_workflow_by_id( - app_model=app_model, workflow_id=workflow_id, session=session - ) - if not workflow: - raise WorkflowNotFoundError(f"Workflow not found with id: {workflow_id}") - return workflow - - if invoke_from == InvokeFrom.DEBUGGER: - # fetch draft workflow by app_model - workflow = workflow_service.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("Workflow not initialized") - else: - # fetch published workflow by app_model - workflow = workflow_service.get_published_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("Workflow not published") - - return workflow + return WorkflowDefinitionStore.prepare_execution_workflow( + app_model, + workflow_id=workflow_id, + draft=invoke_from == InvokeFrom.DEBUGGER, + session=session, + snapshot=snapshot, + ) diff --git a/api/services/app_package_service.py b/api/services/app_package_service.py index a1fda7d5758586..2866e27b8ef096 100644 --- a/api/services/app_package_service.py +++ b/api/services/app_package_service.py @@ -19,8 +19,7 @@ from constants.dsl_version import CURRENT_APP_DSL_VERSION from core.db.session_factory import session_factory from core.tools.entities.tool_entities import ToolProviderType, emoji_icon_adapter -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidator -from models.agent import WorkflowAgentBindingType +from enums.agent import WorkflowAgentBindingType from models.agent_config_entities import WorkflowNodeJobConfig from models.model import App, AppMode from models.tools import ToolLabelBinding, WorkflowToolProvider @@ -41,6 +40,7 @@ validate_icon_references, ) from services.agent.roster_package_reader import RosterAgentPackageReader +from services.agent.workflow_validator import WorkflowAgentNodeValidator from services.app_dsl_service import AppDslService from services.dsl_content import DSL_MAX_SIZE from services.dsl_version import check_version_compatibility diff --git a/api/services/app_preview_details_adapters.py b/api/services/app_preview_details_adapters.py index 2b15403c7e81a6..4404bf0984fc84 100644 --- a/api/services/app_preview_details_adapters.py +++ b/api/services/app_preview_details_adapters.py @@ -10,14 +10,13 @@ from dataclasses import replace from typing import cast, override -from core.agent.tool_configuration import mask_agent_tool_parameters from core.plugin.plugin_service import PluginService from core.tools.entities.tool_entities import ToolProviderType -from core.tools.tool_manager import ToolManager from core.workflow.environment_variables import load_environment_variables from core.workflow.llm_environment_variable import dump_environment_variable from models.model import AppMode from models.provider_ids import GenericProviderID +from models.tool_runtime_contracts import WorkflowToolQueries from services.app_preview_details_service import ( AppPreviewDeletedTool, AppPreviewDetail, @@ -27,10 +26,17 @@ AppPreviewWorkflow, ) from services.app_preview_query_service import AppPreviewRef +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.provider_queries import ToolProviders +from services.tools.tool_manager import ToolManager class AppPreviewDetailsRuntime(AppPreviewDetails): - def __init__(self, *, details: AppPreviewDetailsQuery) -> None: + def __init__( + self, *, details: AppPreviewDetailsQuery, tool_providers: ToolProviders, workflow_queries: WorkflowToolQueries + ) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries self._details: AppPreviewDetailsQuery = details @override @@ -49,6 +55,8 @@ def get_detail(self, *, app: AppPreviewRef, account_id: str, active_workspace_id configuration = { **configuration, "agent_mode": mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=agent_mode, app_id=app.app_id, tenant_id=active_workspace_id, diff --git a/api/services/app_service.py b/api/services/app_service.py index 7c6ae154ad7be1..096c23e09abcc9 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -11,16 +11,16 @@ from pydantic import JsonValue from sqlalchemy import delete, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from configs import dify_config from constants.model_template import default_app_templates from constants.resource_access_token import ResourceAccessTokenResourceType from core.agent.publish_visibility import agent_has_workflow_callable_active_snapshot -from core.agent.tool_configuration import mask_agent_tool_parameters from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.model_manager import ModelManager from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from events.app_event import app_was_created, app_was_deleted, app_was_updated from extensions.application_services.resource_access_token import build_resource_access_token_cleanup_service from extensions.ext_database import db # noqa: F401 @@ -38,18 +38,21 @@ AgentStatus, AgentWorkingResourceStatus, AgentWorkspaceBinding, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig from models.model import App, AppMode, AppModelConfig, IconType, Site, load_annotation_reply_config from models.provider_ids import ModelProviderID from models.skill import AgentSkillBinding +from models.tool_runtime_contracts import WorkflowToolQueries from models.workflow import Workflow +from repositories.agent.home_snapshot_repository import AgentHomeSnapshotRepository +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.app.response import load_app_tool_references from services.agent.errors import AgentAccessNotReadyError, AgentNameConflictError -from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from services.agent.retirement_service import WorkflowAgentRetirementService from services.agent.workspace_service import AgentWorkspaceService +from services.app.response_gateway import AppResponseGateway from services.billing_service import BillingService from services.enterprise import rbac_service as enterprise_rbac_service from services.enterprise.enterprise_service import EnterpriseService @@ -68,6 +71,8 @@ from services.rbac_agent_access_service import initialize_agent_rbac_access from services.system_feature_service import SystemFeatureService from services.tag_service import TagService +from services.tools.agent_configuration import mask_agent_tool_parameters +from services.tools.provider_queries import ToolProviders from tasks.collect_agent_resources_task import enqueue_agent_resource_collection from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task @@ -125,8 +130,17 @@ class AppResponseView: """Expose App response properties through one caller-owned database session.""" def __init__( - self, app: App, *, session: Session, account: Account | None = None, access_mode: str | None = None + self, + app: App, + *, + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, + session: Session, + account: Account | None = None, + access_mode: str | None = None, ) -> None: + self._tool_providers = tool_providers + self._workflow_queries = workflow_queries self._app = app self._session = session self._account = account @@ -154,6 +168,8 @@ def app_model_config(self) -> AppModelConfigResponseView | None: tenant_id = self._account.current_tenant_id assert tenant_id is not None masked_agent_mode = mask_agent_tool_parameters( + tool_providers=self._tool_providers, + workflow_queries=self._workflow_queries, agent_mode=cast(Mapping[str, JsonValue], app_model_config.agent_mode_dict), app_id=self._app.id, tenant_id=tenant_id, @@ -189,7 +205,13 @@ def app_id(self) -> str | None: @property def deleted_tools(self) -> list[Any]: - return self._app.deleted_tools_with_session(session=self._session) + config = self._app.app_model_config_with_session(session=self._session) + references = load_app_tool_references( + session=self._session, + tenant_id=self._app.tenant_id, + tools=config.agent_mode_dict.get("tools", []) if config is not None else [], + ) + return AppResponseGateway.find_deleted_tools(self._app.tenant_id, references) @property def tags(self) -> Sequence[Any]: @@ -857,8 +879,7 @@ def delete_app(self, app: App, *, session: Session, account_id: str | None | Ell binding_id=binding.id, ) retired_binding_ids.append(binding.id) - retired_snapshot_ids = AgentHomeSnapshotService.retire_all_for_agent( - session=session, + retired_snapshot_ids = AgentHomeSnapshotRepository(session).retire_all_for_agent( tenant_id=app.tenant_id, agent_id=backing_agent.id, ) @@ -883,7 +904,9 @@ def delete_app(self, app: App, *, session: Session, account_id: str | None | Ell ) raise - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=app.tenant_id, agent_ids=workflow_agent_ids, account_id=account_id, @@ -985,7 +1008,7 @@ def notify_created_app(*, event: AppEvent, account_id: str, backing_agent_id: st BillingService.clean_billing_info_cache(event.tenant_id) @staticmethod - def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None) -> None: + def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None, sessions: sessionmaker[Session]) -> None: app = deleted.app app_was_deleted.send(app) try: @@ -997,7 +1020,7 @@ def notify_deleted_app(deleted: AppDeletion, *, account_id: str | None) -> None: ) raise - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sessions)).retire_unowned( tenant_id=app.tenant_id, agent_ids=deleted.workflow_agent_ids, account_id=account_id, diff --git a/api/services/app_task_service.py b/api/services/app_task_service.py index 0d14d721e63030..305ca65960cd18 100644 --- a/api/services/app_task_service.py +++ b/api/services/app_task_service.py @@ -15,7 +15,6 @@ from core.app.apps.base_app_queue_manager import AppQueueManager from core.app.apps.execution_coordinator import send_abort_command, set_app_task_stop_flag -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, InvokeFrom from core.app.entities.task_entities import WorkflowFinishStreamResponse from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext @@ -36,6 +35,7 @@ DifyAPISQLAlchemyWorkflowRunRepository, WorkflowTaskOwnerMismatchError, ) +from services.workflow.execution.adapters.response_stream import WorkflowEventStream logger = logging.getLogger(__name__) @@ -326,7 +326,7 @@ def _finish_workflow_stop( files=[], ), ) - topic = MessageBasedAppGenerator.get_response_topic(app_mode, run.id) + topic = WorkflowEventStream.get_response_topic(app_mode, run.id) topic.publish(json.dumps(event.model_dump(mode="json"), ensure_ascii=False).encode()) return True diff --git a/api/services/data_migration/export_service.py b/api/services/data_migration/export_service.py index b5631744791544..6a343622e5a88d 100644 --- a/api/services/data_migration/export_service.py +++ b/api/services/data_migration/export_service.py @@ -7,7 +7,6 @@ import yaml from sqlalchemy.orm import Session -from core.tools.tool_manager import ToolManager from graphon.model_runtime.utils.encoders import jsonable_encoder from models import Account, Tenant from models.account import TenantAccountJoin @@ -26,8 +25,10 @@ ResourceType, ) from services.data_migration.package_service import MigrationPackageService -from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService +from services.tools.mcp_tools_manage_service import MCPToolManageService +from services.tools.provider_queries import ToolProviders +from services.tools.tool_manager import ToolManager +from services.tools.workflow_tools_manage_service import WorkflowToolManageService SUPPORTED_APP_MODES = {"workflow", "advanced-chat"} @@ -114,9 +115,13 @@ class MigrationExportService: def __init__( self, *, + workflow_tools: WorkflowToolManageService, + tool_providers: ToolProviders, package_service: MigrationPackageService | None = None, dependency_discovery_service: DependencyDiscoveryService | None = None, ) -> None: + self._workflow_tools = workflow_tools + self._tool_providers = tool_providers self.package_service = package_service or MigrationPackageService() self.dependency_discovery_service = dependency_discovery_service or DependencyDiscoveryService() @@ -254,6 +259,7 @@ def _export_api_tools( provider=provider_id, tenant_id=tenant_id, mask=not include_secrets, + tool_providers=self._tool_providers, ) if not include_secrets: tool_data.pop("credentials", None) @@ -297,7 +303,7 @@ def _export_workflow_tools( for provider_id in provider_ids: try: - tool_data = WorkflowToolManageService.get_workflow_tool_by_tool_id( + tool_data = self._workflow_tools.get_workflow_tool_by_tool_id( user_id=owner.id, tenant_id=tenant.id, workflow_tool_id=provider_id, diff --git a/api/services/data_migration/import_service.py b/api/services/data_migration/import_service.py index 7b29a77ff60980..3b62e795553775 100644 --- a/api/services/data_migration/import_service.py +++ b/api/services/data_migration/import_service.py @@ -8,26 +8,24 @@ from __future__ import annotations import json -from collections.abc import Iterable +from collections.abc import Callable, Iterable from dataclasses import dataclass -from typing import Any, cast +from typing import Any, Protocol, cast from uuid import UUID import sqlalchemy as sa import yaml from sqlalchemy import or_ -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from configs import dify_config from core.entities.mcp_provider import IdentityMode, MCPAuthentication, MCPConfiguration from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration -from extensions.ext_database import db -from libs.datetime_utils import naive_utc_now +from machinery.context import RequestContext from models import Account, ApiToken, Tenant, TenantAccountJoin, TenantAccountRole from models.enums import ApiTokenType from models.model import App from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider -from services.app_dsl_service import AppDslService from services.data_migration.dependency_discovery_service import DependencyDiscoveryService from services.data_migration.entities import ( ConflictStrategy, @@ -43,11 +41,26 @@ ResourceReportItem, ResourceType, ) -from services.entities.dsl_entities import ImportStatus -from services.tools.legacy_api_tools_manage_service import ApiToolManageService -from services.tools.legacy_mcp_tools_manage_service import MCPToolManageService -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService -from services.workflow_service import WorkflowService +from services.entities.dsl_entities import Import, ImportStatus +from services.tools.api_tools_manage_service import ApiToolManageService +from services.tools.mcp_tools_manage_service import MCPToolManageService +from services.tools.workflow_tools_manage_service import WorkflowToolManageService + + +class WorkflowPublisher(Protocol): + def publish(self, context: RequestContext, app_id: str, *, marked_name: str, marked_comment: str) -> object: ... + + +class WorkflowDslImporter(Protocol): + def import_app( + self, + *, + account: Account, + import_mode: str, + yaml_content: str, + app_id: str | None = None, + import_app_id: str | None = None, + ) -> Import: ... @dataclass(frozen=True) @@ -147,8 +160,18 @@ class MigrationImportService: target_resolver: ImportTargetResolver - def __init__(self, *, target_resolver: ImportTargetResolver | None = None) -> None: + def __init__( + self, + *, + workflows: WorkflowPublisher, + workflow_tools: WorkflowToolManageService, + app_dsl: Callable[[Session], WorkflowDslImporter], + target_resolver: ImportTargetResolver | None = None, + ) -> None: self.target_resolver = target_resolver or ImportTargetResolver() + self._workflow_tools = workflow_tools + self._workflows = workflows + self._app_dsl = app_dsl def import_package(self, request: ImportRequest, *, session: Session) -> ImportResult: target = self.target_resolver.resolve(request, session=session) @@ -323,7 +346,7 @@ def _import_workflow_app( options: ImportOptions, session: Session, ) -> str: - import_service = AppDslService(session) + import_service = self._app_dsl(session) if existing_app is not None: existing_app_id = existing_app.id if dify_config.RBAC_ENABLED: @@ -652,7 +675,7 @@ def _import_workflow_tools( description = self._optional_string(workflow_tool_data.get("description")) or "" privacy_policy = self._optional_string(workflow_tool_data.get("privacy_policy")) or "" if existing is not None: - WorkflowToolManageService.update_workflow_tool( + self._workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=target.tenant_id, workflow_tool_id=existing.id, @@ -668,7 +691,7 @@ def _import_workflow_tools( identifier = existing.id else: import_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else "" - WorkflowToolManageService.create_workflow_tool( + self._workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=target.tenant_id, workflow_app_id=resolved_app_id, @@ -707,24 +730,13 @@ def _ensure_workflow_app_is_published( raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}") if app.workflow_id: return - workflow_service = WorkflowService() - with sessionmaker(db.engine).begin() as session: - app_in_session = session.get(App, app_id) - account_in_session = session.get(Account, account.id) - if app_in_session is None: - raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}") - if account_in_session is None: - raise MigrationDataError(f"Operator account not found: {account.id}") - workflow = workflow_service.publish_workflow( - session=session, - app_model=app_in_session, - account=account_in_session, - marked_name="Migration import", - marked_comment="Published automatically for workflow tool import.", - ) - app_in_session.workflow_id = workflow.id - app_in_session.updated_by = account.id - app_in_session.updated_at = naive_utc_now() + self._workflows.publish( + RequestContext("migration-import", None, account.id, target.tenant_id), + app_id, + marked_name="Migration import", + marked_comment="Published automatically for workflow tool import.", + ) + session.refresh(app) def _import_mcp_tools( self, diff --git a/api/services/installed_app_generation_adapters.py b/api/services/installed_app_generation_adapters.py index 7ab45c157a3f7d..e1eb65e6d0375a 100644 --- a/api/services/installed_app_generation_adapters.py +++ b/api/services/installed_app_generation_adapters.py @@ -9,6 +9,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom from models import Account, App from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.conversation_service import ConversationService @@ -16,13 +17,22 @@ GenerationResponse, InstalledAppGenerationRuntime, ) +from services.workflow.variable_contracts import WorkflowExecutionVariables logger = logging.getLogger(__name__) class AppGenerateServiceRuntime(InstalledAppGenerationRuntime): - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: self._session_factory: sessionmaker[Session] = session_factory + self._variables = variables + self._runtime = runtime @override def generate( @@ -41,6 +51,8 @@ def generate( ) return self._run_generation( lambda session: AppGenerateService.generate( + variables=self._variables, + runtime=self._runtime, session=session, app_model=app, user=account, diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py index 420c86f6c58dec..1dfd6e683e7891 100644 --- a/api/services/knowledge/dataset_service.py +++ b/api/services/knowledge/dataset_service.py @@ -14,7 +14,6 @@ from redis.exceptions import LockNotOwnedError from sqlalchemy import ColumnElement, case, delete, exists, func, select, tuple_, update from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound from configs import dify_config from constants.resource_access_token import ResourceAccessTokenResourceType @@ -42,7 +41,6 @@ ChildChunk, Dataset, DatasetAutoDisableLog, - DatasetCollectionBinding, DatasetPermission, DatasetPermissionEnum, DatasetProcessRule, @@ -66,20 +64,31 @@ from models.source import DataSourceOauthBinding from models.workflow import Workflow from repositories.knowledge import dataset_api_key_bindings -from repositories.knowledge.dataset_read_repository import get_dataset_doc_form, get_latest_dataset_process_rule +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository +from repositories.knowledge.dataset_read_repository import ( + get_dataset_doc_form, + get_latest_dataset_process_rule, +) +from repositories.knowledge.dataset_read_repository import get_datasets_by_ids as load_datasets_by_ids +from repositories.knowledge.document_repository import next_document_position from repositories.knowledge.segment_repository import query_child_chunks +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.document_indexing_proxy.document_indexing_task_proxy import DocumentIndexingTaskProxy from services.document_indexing_proxy.duplicate_document_indexing_task_proxy import DuplicateDocumentIndexingTaskProxy from services.enterprise import rbac_service as enterprise_rbac_service from services.entities.feature_entities import FeatureModel from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, RagPipelineDatasetCreateEntity, ) from services.errors.base import NoPermissionError from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError -from services.errors.dataset import DatasetNameDuplicateError -from services.errors.document import DocumentIndexingError +from services.errors.dataset import DatasetNameDuplicateError, DatasetNotFoundError +from services.errors.document import ( + DocumentAccessDeniedError, + DocumentIndexingError, + DocumentNotFoundError, + DocumentSourceNotFoundError, +) from services.errors.file import FileNotExistsError from services.feature_service import FeatureService from services.file_service import FileService @@ -102,13 +111,11 @@ SegmentMutationService, validate_segment_values, ) -from services.rag_pipeline.rag_pipeline import RagPipelineService from services.rbac import contracts as rbac_contracts from services.tag_application_service import TagTargetQuery from tasks.add_document_to_index_task import add_document_to_index_task from tasks.batch_clean_document_task import batch_clean_document_task from tasks.clean_notion_document_task import clean_notion_document_task -from tasks.deal_dataset_index_update_task import deal_dataset_index_update_task from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task from tasks.document_indexing_update_task import document_indexing_update_task from tasks.recover_document_indexing_task import recover_document_indexing_task @@ -281,21 +288,14 @@ def get_datasets_by_ids( # Check if ids is not empty to avoid WHERE false condition if not ids: return [], 0 - stmt = select(Dataset).where(Dataset.id.in_(ids), Dataset.tenant_id == tenant_id) - - if dify_config.RBAC_ENABLED and accessible_dataset_ids is not None: - requested_dataset_ids = set(ids) - accessible_dataset_ids = [ - dataset_id for dataset_id in accessible_dataset_ids if dataset_id in requested_dataset_ids - ] - accessible_filter: ColumnElement[bool] = Dataset.id.in_(accessible_dataset_ids) - if user: - accessible_filter = sa.or_(Dataset.maintainer == user.id, accessible_filter) - stmt = stmt.where(accessible_filter) - - datasets = paginate_query(stmt, session=session, page=1, per_page=len(ids), max_per_page=len(ids)) - - return datasets.items, datasets.total + datasets = load_datasets_by_ids( + tenant_id, + ids, + session=session, + accessible_dataset_ids=accessible_dataset_ids if dify_config.RBAC_ENABLED else None, + maintainer_id=user.id if user else None, + ) + return datasets, len(datasets) @staticmethod def create_empty_dataset( @@ -767,9 +767,8 @@ def _update_pipeline_knowledge_base_node_data(dataset: Dataset, updata_user_id: return try: - rag_pipeline_service = RagPipelineService(session) - published_workflow = rag_pipeline_service.get_published_workflow(pipeline) - draft_workflow = rag_pipeline_service.get_draft_workflow(pipeline) + published_workflow = WorkflowDefinitionStore.get_published_workflow(pipeline, session=session) + draft_workflow = WorkflowDefinitionStore.get_draft_workflow(pipeline, session=session) # update knowledge nodes def update_knowledge_nodes(workflow_graph: str) -> str: @@ -895,7 +894,7 @@ def _configure_embedding_model_for_high_quality(data, filtered_data, session: Se embedding_model_name = embedding_model.model_name filtered_data["embedding_model"] = embedding_model_name filtered_data["embedding_model_provider"] = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_model.provider, embedding_model_name, session, @@ -1034,7 +1033,7 @@ def _apply_new_embedding_settings(dataset, data, filtered_data, session: Session embedding_model_name = embedding_model.model_name filtered_data["embedding_model"] = embedding_model_name filtered_data["embedding_model_provider"] = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( embedding_model.provider, embedding_model_name, session, @@ -1089,170 +1088,6 @@ def _check_summary_index_setting_model_changed(dataset: Dataset, data: dict[str, return False - @staticmethod - def update_rag_pipeline_dataset_settings( - dataset: Dataset, - knowledge_configuration: KnowledgeConfiguration, - has_published: bool = False, - *, - session: Session, - ): - if not current_user or not current_user.current_tenant_id: - raise ValueError("Current user or current tenant not found") - dataset = session.merge(dataset) - if not has_published: - dataset.chunk_structure = knowledge_configuration.chunk_structure - dataset.indexing_technique = IndexTechniqueType(knowledge_configuration.indexing_technique) - if knowledge_configuration.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, # ignore type error - provider=knowledge_configuration.embedding_model_provider or "", - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model or "", - ) - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - dataset.collection_binding_id = dataset_collection_binding.id - elif knowledge_configuration.indexing_technique == IndexTechniqueType.ECONOMY: - dataset.keyword_number = knowledge_configuration.keyword_number - else: - raise ValueError("Invalid index method") - dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump() - # Update summary_index_setting if provided - if knowledge_configuration.summary_index_setting is not None: - dataset.summary_index_setting = knowledge_configuration.summary_index_setting - session.add(dataset) - else: - if dataset.chunk_structure and dataset.chunk_structure != knowledge_configuration.chunk_structure: - raise ValueError("Chunk structure is not allowed to be updated.") - action = None - if dataset.indexing_technique != knowledge_configuration.indexing_technique: - # if update indexing_technique - if knowledge_configuration.indexing_technique == IndexTechniqueType.ECONOMY: - raise ValueError("Knowledge base indexing technique is not allowed to be updated to economy.") - elif knowledge_configuration.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - action = "add" - # get embedding model setting - try: - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, - provider=knowledge_configuration.embedding_model_provider, - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model, - ) - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - dataset.collection_binding_id = dataset_collection_binding.id - dataset.indexing_technique = IndexTechniqueType(knowledge_configuration.indexing_technique) - except LLMBadRequestError as caught_error: - raise ValueError( - "No Embedding Model available. Please configure a valid provider " - "in the Settings -> Model Provider." - ) from caught_error - except ProviderTokenNotInitError as ex: - raise ValueError(ex.description) from ex - else: - # add default plugin id to both setting sets, to make sure the plugin model provider is consistent - # Skip embedding model checks if not provided in the update request - if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY: - skip_embedding_update = False - try: - # Handle existing model provider - plugin_model_provider = dataset.embedding_model_provider - plugin_model_provider_str = None - if plugin_model_provider: - plugin_model_provider_str = str(ModelProviderID(plugin_model_provider)) - - # Handle new model provider from request - new_plugin_model_provider = knowledge_configuration.embedding_model_provider - new_plugin_model_provider_str = None - if new_plugin_model_provider: - new_plugin_model_provider_str = str(ModelProviderID(new_plugin_model_provider)) - - # Only update embedding model if both values are provided and different from current - if ( - plugin_model_provider_str != new_plugin_model_provider_str - or knowledge_configuration.embedding_model != dataset.embedding_model - ): - action = "update" - model_manager = ModelManager.for_tenant(tenant_id=current_user.current_tenant_id) - embedding_model = None - try: - embedding_model = model_manager.get_model_instance( - tenant_id=current_user.current_tenant_id, - provider=knowledge_configuration.embedding_model_provider, - model_type=ModelType.TEXT_EMBEDDING, - model=knowledge_configuration.embedding_model, - ) - except ProviderTokenNotInitError: - # If we can't get the embedding model, skip updating it - # and keep the existing settings if available - # Skip the rest of the embedding model update - skip_embedding_update = True - if not skip_embedding_update: - if embedding_model: - embedding_model_name = embedding_model.model_name - dataset.embedding_model = embedding_model_name - dataset.embedding_model_provider = embedding_model.provider - dataset_collection_binding = ( - DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, - embedding_model_name, - session, - ) - ) - dataset.collection_binding_id = dataset_collection_binding.id - is_multimodal = DatasetService.check_is_multimodal_model( - current_user.current_tenant_id, - knowledge_configuration.embedding_model_provider, - knowledge_configuration.embedding_model, - ) - dataset.is_multimodal = is_multimodal - except LLMBadRequestError as caught_error: - raise ValueError( - "No Embedding Model available. Please configure a valid provider " - "in the Settings -> Model Provider." - ) from caught_error - except ProviderTokenNotInitError as ex: - raise ValueError(ex.description) from ex - elif dataset.indexing_technique == IndexTechniqueType.ECONOMY: - if dataset.keyword_number != knowledge_configuration.keyword_number: - dataset.keyword_number = knowledge_configuration.keyword_number - dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump() - # Update summary_index_setting if provided - if knowledge_configuration.summary_index_setting is not None: - dataset.summary_index_setting = knowledge_configuration.summary_index_setting - session.add(dataset) - session.commit() - if action: - deal_dataset_index_update_task.delay(dataset.id, action) - @staticmethod def delete_dataset(dataset_id, user, session: Session): dataset = DatasetService.get_dataset(dataset_id, session) @@ -1461,6 +1296,11 @@ class DocumentService: } DOCUMENT_BATCH_DOWNLOAD_ZIP_FILENAME_EXTENSION = ".zip" + @staticmethod + def get_documents_position(dataset_id: str, session: Session) -> int: + """Return the next document position for legacy pipeline execution callers.""" + return next_document_position(dataset_id, session) + @classmethod def normalize_display_status(cls, status: str | None) -> str | None: if not status: @@ -1721,11 +1561,11 @@ def prepare_document_batch_download_zip( """ dataset = DatasetService.get_dataset(dataset_id, session) if not dataset: - raise NotFound("Dataset not found.") + raise DatasetNotFoundError("Dataset not found.") try: DatasetService.check_dataset_permission(dataset, current_user, session) except NoPermissionError as e: - raise Forbidden(str(e)) from e + raise DocumentAccessDeniedError(str(e)) from e upload_files_by_document_id = DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset_id, @@ -1755,12 +1595,12 @@ def _get_upload_file_id_for_upload_file_document( Normalize and validate `Document -> UploadFile` linkage for download flows. """ if document.data_source_type != DataSourceType.UPLOAD_FILE: - raise NotFound(invalid_source_message) + raise DocumentSourceNotFoundError(invalid_source_message) data_source_info: dict[str, Any] = document.data_source_info_dict or {} upload_file_id: str | None = data_source_info.get("upload_file_id") if not upload_file_id: - raise NotFound(missing_file_message) + raise DocumentSourceNotFoundError(missing_file_message) return str(upload_file_id) @@ -1777,7 +1617,7 @@ def _get_upload_file_for_upload_file_document(document: Document, session: Sessi upload_files_by_id = FileService.get_upload_files_by_ids(document.tenant_id, [upload_file_id], session=session) upload_file = upload_files_by_id.get(upload_file_id) if not upload_file: - raise NotFound("Uploaded file not found.") + raise DocumentSourceNotFoundError("Uploaded file not found.") return upload_file @staticmethod @@ -1800,7 +1640,7 @@ def _get_upload_files_by_document_id_for_zip_download( missing_document_ids: set[str] = set(document_id_list) - set(documents_by_id.keys()) if missing_document_ids: - raise NotFound("Document not found.") + raise DocumentNotFoundError("Document not found.") upload_file_ids: list[str] = [] upload_file_ids_by_document_id: dict[str, str] = {} @@ -1816,7 +1656,7 @@ def _get_upload_files_by_document_id_for_zip_download( upload_files_by_id = FileService.get_upload_files_by_ids(tenant_id, upload_file_ids, session=session) missing_upload_file_ids: set[str] = set(upload_file_ids) - set(upload_files_by_id.keys()) if missing_upload_file_ids: - raise NotFound("Only uploaded-file documents can be downloaded as ZIP.") + raise DocumentSourceNotFoundError("Only uploaded-file documents can be downloaded as ZIP.") return { document_id: upload_files_by_id[upload_file_id] @@ -2142,16 +1982,6 @@ def sync_website_document(dataset: Dataset, document: Document, session: Session sync_website_document_indexing_task.delay(dataset.id, document.id) - @staticmethod - def get_documents_position(dataset_id, session: Session): - document = session.scalar( - select(Document).where(Document.dataset_id == dataset_id).order_by(Document.position.desc()).limit(1) - ) - if document: - return document.position + 1 - else: - return 1 - @staticmethod def save_document_with_dataset_id( dataset: Dataset, @@ -2290,7 +2120,7 @@ def save_prepared_documents( dataset_embedding_model_provider = knowledge_config.embedding_model_provider dataset.embedding_model = dataset_embedding_model dataset.embedding_model_provider = dataset_embedding_model_provider - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( dataset_embedding_model_provider, dataset_embedding_model, session ) dataset.collection_binding_id = dataset_collection_binding.id @@ -2368,7 +2198,7 @@ def save_prepared_documents( session.add(dataset_process_rule) session.flush() assert dataset_process_rule - position = DocumentService.get_documents_position(dataset.id, session) + position = next_document_position(dataset.id, session) document_ids = [] duplicate_document_ids = [] removed_notion_ids: list[str] = [] @@ -2613,7 +2443,7 @@ def save_prepared_documents( # dataset_embedding_model_provider = embedding_model.provider # dataset.embedding_model = dataset_embedding_model # dataset.embedding_model_provider = dataset_embedding_model_provider - # dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + # dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( # dataset_embedding_model_provider, dataset_embedding_model # ) # dataset.collection_binding_id = dataset_collection_binding.id @@ -2666,7 +2496,7 @@ def save_prepared_documents( # session.commit() # lock_name = "add_document_lock_dataset_id_{}".format(dataset.id) # with redis_client.lock(lock_name, timeout=600): - # position = DocumentService.get_documents_position(dataset.id) + # position = next_document_position(dataset.id) # document_ids = [] # duplicate_document_ids = [] # if knowledge_config.data_source.info_list.data_source_type == "upload_file": @@ -2954,7 +2784,7 @@ def update_prepared_document( document = DocumentService.get_document(dataset.id, document_data.original_document_id, session=session) if document is None: - raise NotFound("Document not found") + raise DocumentNotFoundError("Document not found") if document.display_status != "available": raise ValueError("Document is not available") # save process rule @@ -3131,7 +2961,7 @@ def build_dataset_for_documents( if knowledge_config.indexing_technique == IndexTechniqueType.HIGH_QUALITY: assert knowledge_config.embedding_model_provider assert knowledge_config.embedding_model - dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( + dataset_collection_binding = DatasetCollectionBindingRepository.get_dataset_collection_binding( knowledge_config.embedding_model_provider, knowledge_config.embedding_model, session, @@ -3868,51 +3698,6 @@ def get_segments_by_document_and_dataset( return session.scalars(query).all() -class DatasetCollectionBindingService: - @classmethod - def get_dataset_collection_binding( - cls, provider_name: str, model_name: str, session: Session, collection_type: str = "dataset" - ) -> DatasetCollectionBinding: - dataset_collection_binding = session.scalar( - select(DatasetCollectionBinding) - .where( - DatasetCollectionBinding.provider_name == provider_name, - DatasetCollectionBinding.model_name == model_name, - DatasetCollectionBinding.type == collection_type, - ) - .order_by(DatasetCollectionBinding.created_at) - .limit(1) - ) - - if not dataset_collection_binding: - dataset_collection_binding = DatasetCollectionBinding( - provider_name=provider_name, - model_name=model_name, - collection_name=Dataset.gen_collection_name_by_id(str(uuid.uuid4())), - type=collection_type, - ) - session.add(dataset_collection_binding) - session.flush() - return dataset_collection_binding - - @classmethod - def get_dataset_collection_binding_by_id_and_type( - cls, collection_binding_id: str, session: Session, collection_type: str = "dataset" - ) -> DatasetCollectionBinding: - dataset_collection_binding = session.scalar( - select(DatasetCollectionBinding) - .where( - DatasetCollectionBinding.id == collection_binding_id, DatasetCollectionBinding.type == collection_type - ) - .order_by(DatasetCollectionBinding.created_at) - .limit(1) - ) - if not dataset_collection_binding: - raise ValueError("Dataset collection binding not found") - - return dataset_collection_binding - - class DatasetPermissionService: @classmethod def get_dataset_partial_member_list(cls, dataset_id, session: Session): diff --git a/api/services/plugin/app_invocation.py b/api/services/plugin/app_invocation.py new file mode 100644 index 00000000000000..07b11c63a6d0d5 --- /dev/null +++ b/api/services/plugin/app_invocation.py @@ -0,0 +1,368 @@ +import uuid +from collections.abc import Generator, Mapping +from typing import Any, Protocol, cast + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict +from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig +from core.db.session_factory import create_session +from core.plugin.backwards_invocation.base import BaseBackwardsInvocation +from extensions.ext_database import db +from models import Account, TenantAccountJoin +from models.annotation_reply import AnnotationReplies +from models.model import ( + App, + AppMode, + AppModelConfig, + AppModelConfigDict, + EndUser, + load_annotation_reply_config, +) +from models.workflow import Workflow +from repositories.workflow.definition_repository import workflow_snapshot +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.adapters.completion import CompletionAppGenerator +from services.app.generation.ports import ChatRecords +from services.app.generation.runtime import AppGenerationRuntime +from services.knowledge.retrieval.ports import DatasetRetrievalFactory +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow_execution_identity import build_workflow_execution_identity + + +class AppScopedEndUserProvisioner(Protocol): + def get_or_create_end_user( + self, + tenant_id: str, + app_id: str, + user_id: str | None = None, + ) -> EndUser: ... + + +class PluginAppBackwardsInvocation(BaseBackwardsInvocation): + @classmethod + def fetch_app_info(cls, app_id: str, tenant_id: str) -> Mapping: + """ + Fetch app info + """ + app = cls._get_app(app_id, tenant_id) + + """Retrieve app parameters.""" + if app.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: + workflow = cls._get_workflow(app) + if workflow is None: + raise ValueError("unexpected app type") + + features_dict: dict[str, Any] = workflow.features_dict + user_input_form = workflow.user_input_form(to_old_structure=True) + else: + app_model_config_dict = cls._get_app_model_config_dict(app) + if app_model_config_dict is None: + raise ValueError("unexpected app type") + + features_dict = cast(dict[str, Any], app_model_config_dict) + + user_input_form = features_dict.get("user_input_form", []) + + return { + "data": get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form), + } + + @classmethod + def invoke_app( + cls, + app_id: str, + user_id: str, + tenant_id: str, + conversation_id: str | None, + query: str | None, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + end_users: AppScopedEndUserProvisioner, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str] | Mapping: + """ + invoke app + """ + app = cls._get_app(app_id, tenant_id) + if not user_id: + user = end_users.get_or_create_end_user(app.tenant_id, app.id, None) + else: + try: + user = cls._get_user(user_id, app) + except ValueError: + # Plugins such as WeCom Bot pass external sender IDs rather than EndUser UUIDs. + user = end_users.get_or_create_end_user(app.tenant_id, app.id, user_id) + + conversation_id = conversation_id or "" + + match app.mode: + case AppMode.ADVANCED_CHAT | AppMode.AGENT_CHAT | AppMode.CHAT: + if not query: + raise ValueError("missing query") + + return cls.invoke_chat_app( + app, user, conversation_id, query, stream, inputs, files, session, runtime=runtime + ) + case AppMode.WORKFLOW: + workflow = cls._get_workflow(app) + if not workflow: + raise ValueError("unexpected app type") + return cls.invoke_workflow_app(app, workflow, user, stream, inputs, files, runtime=runtime) + case AppMode.COMPLETION: + return cls.invoke_completion_app( + app, + user, + stream, + inputs, + files, + session, + annotations=runtime.annotation_replies, + retrieval=runtime.retrieval, + records=runtime.chat_records, + ) + case _: + raise ValueError("unexpected app type") + + @classmethod + def invoke_chat_app( + cls, + app: App, + user: Account | EndUser, + conversation_id: str, + query: str, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str] | Mapping: + """ + invoke chat app + """ + match app.mode: + case AppMode.ADVANCED_CHAT: + workflow = cls._get_workflow(app) + if not workflow: + raise ValueError("unexpected app type") + + pause_config = PauseStateLayerConfig( + session_factory=db.engine, + state_owner_user_id=workflow.created_by, + ) + + return AdvancedChatAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + ).generate( + app_model=app, + workflow=workflow, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + workflow_run_id=str(uuid.uuid4()), + streaming=stream, + pause_state_config=pause_config, + ) + case AppMode.AGENT_CHAT: + return AgentChatAppGenerator( + annotations=runtime.annotation_replies, + records=runtime.chat_records, + dataset_tools=runtime.dataset_tools, + tool_invoker=runtime.agent_tool_invoker, + workflow_runtime=runtime, + ).generate( + app_model=app, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + case AppMode.CHAT: + return ChatAppGenerator( + retrieval=runtime.retrieval, annotations=runtime.annotation_replies, records=runtime.chat_records + ).generate( + app_model=app, + user=user, + args={ + "inputs": inputs, + "query": query, + "files": files, + "conversation_id": conversation_id, + }, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + case _: + raise ValueError("unexpected app type") + + @classmethod + def invoke_workflow_app( + cls, + app: App, + workflow: WorkflowSnapshot, + user: EndUser | Account, + stream: bool, + inputs: Mapping, + files: list[dict], + *, + runtime: AppGenerationRuntime, + ) -> Generator[Mapping | str] | Mapping: + """ + invoke workflow app + """ + pause_config = PauseStateLayerConfig( + session_factory=db.engine, + state_owner_user_id=workflow.created_by, + ) + + return WorkflowAppGenerator( + runtime=runtime, + execution_driver=runtime.execution_driver, + ).generate( + app_model=app, + workflow=workflow, + user=user, + execution_identity=build_workflow_execution_identity(user), + args={"inputs": inputs, "files": files}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + call_depth=1, + pause_state_config=pause_config, + ) + + @classmethod + def invoke_completion_app( + cls, + app: App, + user: EndUser | Account, + stream: bool, + inputs: Mapping, + files: list[dict], + session: Session, + *, + records: ChatRecords, + annotations: AnnotationReplies, + retrieval: DatasetRetrievalFactory, + ) -> Generator[Mapping | str] | Mapping: + """ + invoke completion app + """ + return CompletionAppGenerator(retrieval=retrieval, annotations=annotations, records=records).generate( + app_model=app, + user=user, + args={"inputs": inputs, "files": files}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=stream, + session=session, + ) + + @classmethod + def _get_user(cls, user_id: str, app: App) -> EndUser | Account: + """ + get the user by user id + """ + with create_session() as session: + stmt = select(EndUser).where( + EndUser.id == user_id, + EndUser.tenant_id == app.tenant_id, + EndUser.app_id == app.id, + ) + user = session.scalar(stmt) + if not user: + stmt = select(EndUser).where( + EndUser.session_id == user_id, + EndUser.tenant_id == app.tenant_id, + EndUser.app_id == app.id, + ) + user = session.scalar(stmt) + if not user: + stmt = select(Account).where( + Account.id == user_id, + Account.id == TenantAccountJoin.account_id, + TenantAccountJoin.tenant_id == app.tenant_id, + ) + user = session.scalar(stmt) + if user: + session.expunge(user) + + if not user: + raise ValueError("user not found") + + return user + + @classmethod + def _get_app(cls, app_id: str, tenant_id: str) -> App: + """ + get app + """ + try: + with create_session() as session: + app = session.scalar(select(App).where(App.id == app_id, App.tenant_id == tenant_id).limit(1)) + if app: + session.expunge(app) + except Exception as e: + raise ValueError("app not found") from e + + if not app: + raise ValueError("app not found") + + return app + + @classmethod + def _get_workflow(cls, app: App) -> WorkflowSnapshot | None: + """ + get workflow with an owned session, additionally scoped by tenant and app + + `App.workflow_with_session` resolves by `workflow_id` alone; a plugin + invocation must also confirm the workflow belongs to this tenant and app. + """ + if not app.workflow_id: + return None + + with create_session() as session: + workflow = session.scalar( + select(Workflow) + .where(Workflow.id == app.workflow_id, Workflow.tenant_id == app.tenant_id, Workflow.app_id == app.id) + .limit(1) + ) + return workflow_snapshot(workflow) if workflow is not None else None + + @classmethod + def _get_app_model_config_dict(cls, app: App) -> AppModelConfigDict | None: + """ + get app model config features without relying on request-scoped session-backed model properties + """ + if not app.app_model_config_id: + return None + + with create_session() as session: + app_model_config = session.scalar( + select(AppModelConfig) + .where(AppModelConfig.id == app.app_model_config_id, AppModelConfig.app_id == app.id) + .limit(1) + ) + if app_model_config is None: + return None + + annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) + return app_model_config.to_dict(annotation_reply=annotation_reply) diff --git a/api/services/rag_pipeline/execution_service.py b/api/services/rag_pipeline/execution_service.py index 80175a103ac546..6e4af16e2ac01d 100644 --- a/api/services/rag_pipeline/execution_service.py +++ b/api/services/rag_pipeline/execution_service.py @@ -62,10 +62,11 @@ WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, ) +from repositories.workflow.definition_repository import workflow_snapshot from services.data_source.provider_service import DatasourceProviderService from services.workflow.execution.adapters.workflow_entry import WorkflowEntry from services.workflow.execution.ports import WorkflowRuntime -from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow.variable_contracts import WorkflowExecutionVariables, environment_variable_values logger = logging.getLogger(__name__) @@ -126,9 +127,10 @@ def run_draft_workflow_node( Run draft workflow node """ # fetch draft workflow by app_model - draft_workflow = self._workflows.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: + draft_workflow_model = self._workflows.get_draft_workflow(pipeline=pipeline) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) # run draft workflow node start_at = time.perf_counter() @@ -151,7 +153,7 @@ def run_draft_workflow_node( variable_pool=_build_seeded_variable_pool( build_bootstrap_variables( system_variables=default_system_variables(), - environment_variables=draft_workflow.environment_variables, + environment_variables=environment_variable_values(draft_workflow), ) ), variable_loader=self._variables.workflow_loader(draft_workflow, account.id), @@ -638,9 +640,10 @@ def set_datasource_variables(self, pipeline: Pipeline, args: dict[str, Any], cur """ # fetch draft workflow by app_model - draft_workflow = self._workflows.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: + draft_workflow_model = self._workflows.get_draft_workflow(pipeline=pipeline) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) # run draft workflow node start_at = time.perf_counter() diff --git a/api/services/rag_pipeline/pipeline_generate_service.py b/api/services/rag_pipeline/pipeline_generate_service.py index 12e85611b040a1..aa36b4f2a9e9a0 100644 --- a/api/services/rag_pipeline/pipeline_generate_service.py +++ b/api/services/rag_pipeline/pipeline_generate_service.py @@ -3,17 +3,15 @@ from sqlalchemy.orm import Session -from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from models.dataset import Pipeline from models.enums import IndexingStatus -from models.model import Account, App, EndUser -from models.workflow import Workflow +from models.model import Account, EndUser from repositories.knowledge.dataset_read_repository import get_pipeline_dataset +from services.app.generation.response import convert_to_event_stream from services.knowledge.dataset_service import DocumentService from services.knowledge.resource_scope import DatasetRef, DocumentRef -from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator class PipelineGenerateService: @@ -39,17 +37,17 @@ def generate( :return: """ try: - workflow = cls._get_workflow(pipeline, invoke_from, session) + workflow = generator.load_workflow(pipeline, invoke_from) if original_document_id := args.get("original_document_id"): dataset = get_pipeline_dataset(pipeline, session) if dataset is None or dataset.tenant_id != pipeline.tenant_id: raise ValueError("Pipeline dataset is required") - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document(original_document_id) + document_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id).document( + original_document_id + ) cls.update_document_status(document_ref, session=session) - return PipelineGenerator.convert_to_event_stream( + return convert_to_event_stream( generator.generate( - session=session, pipeline=pipeline, workflow=workflow, user=user, @@ -64,13 +62,16 @@ def generate( except Exception: raise - @staticmethod - def _get_max_active_requests(app_model: App) -> int: - app_limit = app_model.max_active_requests or dify_config.APP_DEFAULT_ACTIVE_REQUESTS - config_limit = dify_config.APP_MAX_ACTIVE_REQUESTS - # Filter out infinite (0) values and return the minimum, or 0 if both are infinite - limits = [limit for limit in [app_limit, config_limit] if limit > 0] - return min(limits) if limits else 0 + @classmethod + def update_document_status(cls, document_ref: DocumentRef, *, session: Session) -> None: + """Set an owner-bound pipeline document to waiting before regeneration.""" + document = next( + iter(DocumentService.get_documents_by_ids(document_ref.dataset, [document_ref.document_id], session)), None + ) + if document is None: + raise ValueError("Pipeline document not found") + document.indexing_status = IndexingStatus.WAITING + session.add(document) @classmethod def generate_single_iteration( @@ -79,13 +80,12 @@ def generate_single_iteration( user: Account, node_id: str, args: Any, - session: Session, streaming: bool = True, *, generator: PipelineGenerator, ): - workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session) - return PipelineGenerator.convert_to_event_stream( + workflow = generator.load_workflow(pipeline, InvokeFrom.DEBUGGER) + return convert_to_event_stream( generator.single_iteration_generate( pipeline=pipeline, workflow=workflow, @@ -93,7 +93,6 @@ def generate_single_iteration( user=user, args=args, streaming=streaming, - session=session, ) ) @@ -104,13 +103,12 @@ def generate_single_loop( user: Account, node_id: str, args: Any, - session: Session, streaming: bool = True, *, generator: PipelineGenerator, ): - workflow = cls._get_workflow(pipeline, InvokeFrom.DEBUGGER, session) - return PipelineGenerator.convert_to_event_stream( + workflow = generator.load_workflow(pipeline, InvokeFrom.DEBUGGER) + return convert_to_event_stream( generator.single_loop_generate( pipeline=pipeline, workflow=workflow, @@ -118,41 +116,5 @@ def generate_single_loop( user=user, args=args, streaming=streaming, - session=session, ) ) - - @classmethod - def _get_workflow(cls, pipeline: Pipeline, invoke_from: InvokeFrom, session: Session) -> Workflow: - """ - Get workflow - :param pipeline: pipeline - :param invoke_from: invoke from - :return: - """ - rag_pipeline_service = RagPipelineService(session) - if invoke_from == InvokeFrom.DEBUGGER: - # fetch draft workflow by app_model - workflow = rag_pipeline_service.get_draft_workflow(pipeline=pipeline) - - if not workflow: - raise ValueError("Workflow not initialized") - else: - # fetch published workflow by app_model - workflow = rag_pipeline_service.get_published_workflow(pipeline=pipeline) - - if not workflow: - raise ValueError("Workflow not published") - - return workflow - - @classmethod - def update_document_status(cls, document_ref: DocumentRef, *, session: Session) -> None: - """Set a document in the owner-bound dataset to waiting.""" - document = next( - iter(DocumentService.get_documents_by_ids(document_ref.dataset, [document_ref.document_id], session)), None - ) - if document is None: - raise ValueError("Pipeline document not found") - document.indexing_status = IndexingStatus.WAITING - session.add(document) diff --git a/api/services/rag_pipeline/rag_pipeline.py b/api/services/rag_pipeline/rag_pipeline.py index 29b22ab31a40d9..c9de59c2ae4de0 100644 --- a/api/services/rag_pipeline/rag_pipeline.py +++ b/api/services/rag_pipeline/rag_pipeline.py @@ -2,59 +2,18 @@ import logging import re import threading -import time -from collections.abc import Callable, Generator, Mapping, Sequence -from datetime import UTC, datetime -from typing import Any, cast -from uuid import uuid4 +from collections.abc import Mapping, Sequence +from typing import Any from sqlalchemy import func, select from sqlalchemy.orm import Session, sessionmaker import contexts from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom -from core.datasource.datasource_manager import DatasourceManager -from core.datasource.entities.datasource_entities import ( - DatasourceMessage, - DatasourceProviderType, - GetOnlineDocumentPageContentRequest, - OnlineDocumentPagesMessage, - OnlineDriveBrowseFilesRequest, - OnlineDriveBrowseFilesResponse, - WebsiteCrawlMessage, -) -from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin -from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin -from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin from core.helper import marketplace -from core.rag.entities import DatasourceCompletedEvent, DatasourceErrorEvent, DatasourceProcessingEvent -from core.repositories.factory import DifyCoreRepositoryFactory -from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository -from core.workflow.llm_environment_variable import validate_llm_environment_model_references -from core.workflow.node_factory import LATEST_VERSION, get_node_type_classes_mapping -from core.workflow.system_variables import ( - SystemVariableKey, - build_bootstrap_variables, - build_system_variables, - default_system_variables, - get_system_segment, -) -from core.workflow.variable_pool_initializer import add_variables_to_pool -from core.workflow.workflow_entry import WorkflowEntry -from enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace -from extensions.ext_database import db -from graphon.engine_events import NodeEvent, NodeRunFailedEvent, NodeRunSucceededEvent -from graphon.entities import WorkflowNodeExecution -from graphon.enums import BuiltinNodeTypes, ErrorStrategy, NodeType, WorkflowNodeExecutionStatus -from graphon.errors import WorkflowNodeRunFailedError -from graphon.node_events import NodeRunResult -from graphon.nodes.base.node import Node -from graphon.nodes.container_effects import ContainerAwaitRequest +from graphon.enums import BuiltinNodeTypes, NodeType from graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, build_http_request_config -from graphon.runtime import VariablePool -from graphon.variables.variables import Variable, VariableBase from libs.infinite_scroll_pagination import InfiniteScrollPagination from libs.login import resolve_account_fallback, resolve_tenant_id_fallback from models import Account @@ -66,47 +25,37 @@ PipelineCustomizedTemplate, PipelineRecommendedPlugin, ) -from models.enums import IndexingStatus, WorkflowRunTriggeredFrom +from models.enums import WorkflowRunTriggeredFrom from models.model import EndUser from models.workflow import ( Workflow, WorkflowNodeExecutionModel, - WorkflowNodeExecutionTriggeredFrom, WorkflowRun, - WorkflowType, ) from repositories.factory import DifyAPIRepositoryFactory -from repositories.knowledge.dataset_read_repository import get_pipeline_dataset +from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_record, workflow_snapshot from services.credentials.query import CredentialQuery from services.data_source.provider_service import DatasourceProviderService from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, PipelineTemplateInfoEntity, ) -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.rag_pipeline import RagPipelineResourceNotFoundError -from services.knowledge.resource_scope import DatasetRef from services.rag_pipeline.pipeline_template.pipeline_template_factory import PipelineTemplateRetrievalFactory from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService -from services.tools.legacy_builtin_tools_manage_service import BuiltinToolManageService -from services.workflow_draft_variable_service import DraftVariableSaver, DraftVarLoader +from services.tools.builtin_tools_manage_service import BuiltinToolManageService +from services.tools.provider_queries import ToolProviders +from services.workflow.contracts import WorkflowRecord +from services.workflow.execution.adapters.node_factory import LATEST_VERSION, get_node_type_classes_mapping +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, ) from services.workflow_ref_service import WorkflowRef -from services.workflow_restore import apply_published_workflow_snapshot_to_draft -from services.workflow_run_agg import WorkflowRunAgg logger = logging.getLogger(__name__) -def _build_seeded_variable_pool(variables: Sequence[Variable]) -> VariablePool: - variable_pool = VariablePool() - add_variables_to_pool(variable_pool, variables) - return variable_pool - - class RagPipelineService: _session: Session @@ -114,7 +63,7 @@ def __init__(self, session: Session, session_maker: sessionmaker | None = None): """Initialize RagPipelineService with repository dependencies.""" self._session = session if session_maker is None: - session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + session_maker = sessionmaker(bind=session.get_bind(), expire_on_commit=False) self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) @@ -270,60 +219,22 @@ def get_draft_workflow(self, pipeline: Pipeline) -> Workflow | None: """ Get draft workflow """ - # fetch draft workflow by rag pipeline - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == "draft", - ) - .limit(1) - ) + return WorkflowDefinitionStore.get_draft_workflow(pipeline, session=self._session) - # return draft workflow - return workflow + def get_draft_workflow_record(self, pipeline: Pipeline) -> WorkflowRecord | None: + workflow = self.get_draft_workflow(pipeline) + return workflow_record(workflow, self._session) if workflow is not None else None def get_published_workflow(self, pipeline: Pipeline) -> Workflow | None: """ Get published workflow """ - if not pipeline.workflow_id: - return None + return WorkflowDefinitionStore.get_published_workflow(pipeline, session=self._session) - # fetch published workflow by workflow_id - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.id == pipeline.workflow_id, - ) - .limit(1) - ) - - return workflow - - def get_published_workflow_by_id(self, pipeline: Pipeline, workflow_id: str) -> Workflow | None: - """Fetch and lock a published Workflow snapshot for restoration. - - The source lock is held until the service transaction ends, preventing - concurrent deletion while restore copies its Workflow snapshot fields. - """ - workflow = self._session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.id == workflow_id, - ) - .limit(1) - .with_for_update() - ) - if workflow and workflow.version == Workflow.VERSION_DRAFT: - raise IsDraftWorkflowError("source workflow must be published") - return workflow + def get_published_workflow_record(self, pipeline: Pipeline) -> WorkflowRecord | None: + workflow = self.get_published_workflow(pipeline) + return workflow_record(workflow, self._session) if workflow is not None else None def get_all_published_workflow( self, @@ -334,188 +245,20 @@ def get_all_published_workflow( limit: int, user_id: str | None, named_only: bool = False, - ) -> tuple[Sequence[Workflow], bool]: + ) -> tuple[Sequence[WorkflowRecord], bool]: """ Get published workflow with pagination """ - if not pipeline.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where(Workflow.app_id == pipeline.id) - .order_by(Workflow.version.desc()) - .limit(limit + 1) - .offset((page - 1) * limit) - ) - - if user_id: - stmt = stmt.where(Workflow.created_by == user_id) - - if named_only: - stmt = stmt.where(Workflow.marked_name != "") - - workflows = session.scalars(stmt).all() - - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more - - def sync_draft_workflow( - self, - *, - pipeline: Pipeline, - graph: dict[str, Any], - unique_hash: str | None, - account: Account, - environment_variables: Sequence[VariableBase], - conversation_variables: Sequence[VariableBase], - rag_pipeline_variables: list, - ) -> Workflow: - """ - Sync draft workflow - :raises WorkflowHashNotEqualError - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(pipeline=pipeline) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # create draft workflow if not found - if not workflow: - workflow = Workflow( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - _features="{}", - type=WorkflowType.RAG_PIPELINE, - version="draft", - graph=json.dumps(graph), - created_by=account.id, - _environment_variables=environment_variables, - _conversation_variables=conversation_variables, - _rag_pipeline_variables=rag_pipeline_variables, - ) - self._session.add(workflow) - self._session.flush() - pipeline.workflow_id = workflow.id - # update draft workflow if found - else: - workflow.graph = json.dumps(graph) - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - workflow.environment_variables = environment_variables - workflow.conversation_variables = conversation_variables - workflow.rag_pipeline_variables = rag_pipeline_variables - # commit db session changes - self._session.commit() - - # trigger workflow events TODO - # app_draft_workflow_was_synced.send(pipeline, synced_draft_workflow=workflow) - - # return draft workflow - return workflow - - def restore_published_workflow_to_draft( - self, - *, - pipeline: Pipeline, - workflow_id: str, - account: Account, - ) -> Workflow: - """Restore a published pipeline workflow snapshot into the draft workflow. - - Pipelines reuse the shared draft-restore field copy helper, but still own - the pipeline-specific flush/link step that wires a newly created draft - back onto ``pipeline.workflow_id``. The source version remains locked - through snapshot-field copy and commit. - """ - source_workflow = self.get_published_workflow_by_id(pipeline=pipeline, workflow_id=workflow_id) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - draft_workflow, is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=lambda: datetime.now(UTC).replace(tzinfo=None), - ) - - if is_new_draft: - self._session.add(draft_workflow) - self._session.flush() - pipeline.workflow_id = draft_workflow.id - - self._session.commit() - - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - pipeline: Pipeline, - account: Account, - ) -> Workflow: - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == "draft", + workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( + session=session, + app_model=pipeline, + page=page, + limit=limit, + user_id=user_id, + named_only=named_only, ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") - validate_llm_environment_model_references( - graph=draft_workflow.graph_dict, - environment_variables=draft_workflow.environment_variables, - ) - - # create new workflow - workflow = Workflow.new( - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - type=draft_workflow.type, - version=str(datetime.now(UTC).replace(tzinfo=None)), - graph=draft_workflow.graph, - features=draft_workflow.features, - created_by=account.id, - environment_variables=draft_workflow.environment_variables, - conversation_variables=draft_workflow.conversation_variables, - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - marked_name="", - marked_comment="", - ) - # commit db session changes - session.add(workflow) - - graph = workflow.graph_dict - nodes = graph.get("nodes", []) - # dataset_service imports RagPipelineService, so defer this reverse dependency. - from services.knowledge.dataset_service import DatasetService - - for node in nodes: - if node.get("data", {}).get("type") == "knowledge-index": - knowledge_configuration = node.get("data", {}) - knowledge_configuration = KnowledgeConfiguration.model_validate(knowledge_configuration) - - # update dataset - dataset = get_pipeline_dataset(pipeline, session=session) - if not dataset: - raise ValueError("Dataset not found") - DatasetService.update_rag_pipeline_dataset_settings( - session=session, - dataset=dataset, - knowledge_configuration=knowledge_configuration, - has_published=pipeline.is_published, - ) - # return new workflow - return workflow + return [workflow_record(workflow, session) for workflow in workflows], has_more def get_default_block_configs(self) -> list[dict]: """ @@ -578,517 +321,6 @@ def get_default_block_config( return default_config - def run_draft_workflow_node( - self, pipeline: Pipeline, node_id: str, user_inputs: dict[str, Any], account: Account - ) -> WorkflowNodeExecutionModel | None: - """ - Run draft workflow node - """ - # fetch draft workflow by app_model - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - start_at = time.perf_counter() - node_config = draft_workflow.get_node_config_by_id(node_id) - - eclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - if eclosing_node_type_and_id: - _, enclosing_node_id = eclosing_node_type_and_id - else: - enclosing_node_id = None - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.single_step_run( - execution_driver=WorkflowRunAgg.run, - workflow=draft_workflow, - node_id=node_id, - user_inputs=user_inputs, - user_id=account.id, - variable_pool=_build_seeded_variable_pool( - build_bootstrap_variables( - system_variables=default_system_variables(), - environment_variables=draft_workflow.environment_variables, - ) - ), - variable_loader=DraftVarLoader( - engine=db.engine, - app_id=pipeline.id, - tenant_id=pipeline.tenant_id, - user_id=account.id, - ), - ), - start_at=start_at, - tenant_id=pipeline.tenant_id, - node_id=node_id, - ) - workflow_node_execution.workflow_id = draft_workflow.id - - # Create repository and save the node execution - - repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=db.engine, - tenant_id=pipeline.tenant_id, - user=account, - app_id=pipeline.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, - ) - repository.save(workflow_node_execution) - - # Convert node_execution to WorkflowNodeExecution after save - workflow_node_execution_db_model = self._node_execution_service_repo.get_execution_by_id( - workflow_node_execution.id - ) - - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - node_id=workflow_node_execution.node_id, - node_type=workflow_node_execution.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=workflow_node_execution.id, - user=account, - ) - draft_var_saver.save( - process_data=workflow_node_execution.process_data, - outputs=workflow_node_execution.outputs, - ) - if isinstance(workflow_node_execution_db_model, WorkflowNodeExecutionModel): - enqueue_draft_node_execution_trace( - execution=workflow_node_execution_db_model, - outputs=workflow_node_execution.outputs, - workflow_execution_id=None, - user_id=account.id, - ) - return workflow_node_execution_db_model - - def run_datasource_workflow_node( - self, - pipeline: Pipeline, - node_id: str, - user_inputs: dict[str, Any], - account: Account, - datasource_type: str, - is_published: bool, - credential_id: str | None = None, - *, - datasource_providers: DatasourceProviderService, - ) -> Generator[Mapping[str, Any]]: - """ - Run published workflow datasource - """ - try: - if is_published: - # fetch published workflow by app_model - workflow = self.get_published_workflow(pipeline=pipeline) - else: - workflow = self.get_draft_workflow(pipeline=pipeline) - if not workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - datasource_node_data = None - datasource_nodes = workflow.graph_dict.get("nodes", []) - for datasource_node in datasource_nodes: - if datasource_node.get("id") == node_id: - datasource_node_data = datasource_node.get("data", {}) - break - if not datasource_node_data: - raise ValueError("Datasource node data not found") - - variables_map = {} - - datasource_parameters = datasource_node_data.get("datasource_parameters", {}) - for key, value in datasource_parameters.items(): - param_value = value.get("value") - - match param_value: - case None | "" | [] | {}: - variables_map[key] = param_value - case str(): - # handle string type parameter value, check if it contains variable reference pattern - pattern = r"\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z0-9_][a-zA-Z0-9_]{0,29}){1,10})#\}\}" - match_result = re.match(pattern, param_value) - if match_result: - # extract variable path and try to get value from user inputs - full_path = match_result.group(1) - last_part = full_path.split(".")[-1] - variables_map[key] = user_inputs.get(last_part, param_value) - else: - variables_map[key] = param_value - case list() if param_value: - # handle list type parameter value, check if the last element is in user inputs - last_part = param_value[-1] - variables_map[key] = user_inputs.get(last_part, param_value) - case _: - # other type directly use original value - variables_map[key] = param_value - - datasource_runtime = DatasourceManager.get_datasource_runtime( - provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}", - datasource_name=datasource_node_data.get("datasource_name"), - tenant_id=pipeline.tenant_id, - datasource_type=DatasourceProviderType(datasource_type), - ) - credentials = datasource_providers.get_datasource_credentials( - tenant_id=pipeline.tenant_id, - provider=datasource_node_data.get("provider_name"), - plugin_id=datasource_node_data.get("plugin_id"), - credential_id=credential_id, - ) - if credentials: - datasource_runtime.runtime.credentials = credentials - match datasource_type: - case DatasourceProviderType.ONLINE_DOCUMENT: - datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime) - online_document_result: Generator[OnlineDocumentPagesMessage] = ( - datasource_runtime.get_online_document_pages( - user_id=account.id, - datasource_parameters=user_inputs, - provider_type=datasource_runtime.datasource_provider_type(), - ) - ) - start_time = time.time() - start_event = DatasourceProcessingEvent( - total=0, - completed=0, - ) - yield start_event.model_dump() - try: - for online_document_message in online_document_result: - end_time = time.time() - online_document_event = DatasourceCompletedEvent( - data=online_document_message.result, time_consuming=round(end_time - start_time, 2) - ) - yield online_document_event.model_dump() - except Exception as e: - logger.exception("Error during online document.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - case DatasourceProviderType.ONLINE_DRIVE: - datasource_runtime = cast(OnlineDriveDatasourcePlugin, datasource_runtime) - online_drive_result: Generator[OnlineDriveBrowseFilesResponse] = ( - datasource_runtime.online_drive_browse_files( - user_id=account.id, - request=OnlineDriveBrowseFilesRequest( - bucket=user_inputs.get("bucket"), - prefix=user_inputs.get("prefix", ""), - max_keys=user_inputs.get("max_keys", 20), - next_page_parameters=user_inputs.get("next_page_parameters"), - ), - provider_type=datasource_runtime.datasource_provider_type(), - ) - ) - start_time = time.time() - start_event = DatasourceProcessingEvent( - total=0, - completed=0, - ) - yield start_event.model_dump() - for online_drive_message in online_drive_result: - end_time = time.time() - online_drive_event = DatasourceCompletedEvent( - data=online_drive_message.result, - time_consuming=round(end_time - start_time, 2), - total=None, - completed=None, - ) - yield online_drive_event.model_dump() - case DatasourceProviderType.WEBSITE_CRAWL: - datasource_runtime = cast(WebsiteCrawlDatasourcePlugin, datasource_runtime) - website_crawl_result: Generator[WebsiteCrawlMessage] = datasource_runtime.get_website_crawl( - user_id=account.id, - datasource_parameters=variables_map, - provider_type=datasource_runtime.datasource_provider_type(), - ) - start_time = time.time() - try: - for website_crawl_message in website_crawl_result: - end_time = time.time() - crawl_event: DatasourceCompletedEvent | DatasourceProcessingEvent - if website_crawl_message.result.status == "completed": - crawl_event = DatasourceCompletedEvent( - data=website_crawl_message.result.web_info_list or [], - total=website_crawl_message.result.total, - completed=website_crawl_message.result.completed, - time_consuming=round(end_time - start_time, 2), - ) - else: - crawl_event = DatasourceProcessingEvent( - total=website_crawl_message.result.total, - completed=website_crawl_message.result.completed, - ) - yield crawl_event.model_dump() - except Exception as e: - logger.exception("Error during website crawl.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - case _: - raise ValueError(f"Unsupported datasource provider: {datasource_runtime.datasource_provider_type}") - except Exception as e: - logger.exception("Error in run_datasource_workflow_node.") - yield DatasourceErrorEvent(error=str(e)).model_dump() - - def run_datasource_node_preview( - self, - pipeline: Pipeline, - node_id: str, - user_inputs: dict[str, Any], - account: Account, - datasource_type: str, - is_published: bool, - credential_id: str | None = None, - *, - datasource_providers: DatasourceProviderService, - ) -> Mapping[str, Any]: - """ - Run published workflow datasource - """ - try: - if is_published: - # fetch published workflow by app_model - workflow = self.get_published_workflow(pipeline=pipeline) - else: - workflow = self.get_draft_workflow(pipeline=pipeline) - if not workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - datasource_node_data = None - datasource_nodes = workflow.graph_dict.get("nodes", []) - for datasource_node in datasource_nodes: - if datasource_node.get("id") == node_id: - datasource_node_data = datasource_node.get("data", {}) - break - if not datasource_node_data: - raise ValueError("Datasource node data not found") - - datasource_parameters = datasource_node_data.get("datasource_parameters", {}) - for key, value in datasource_parameters.items(): - if not user_inputs.get(key): - user_inputs[key] = value["value"] - - datasource_runtime = DatasourceManager.get_datasource_runtime( - provider_id=f"{datasource_node_data.get('plugin_id')}/{datasource_node_data.get('provider_name')}", - datasource_name=datasource_node_data.get("datasource_name"), - tenant_id=pipeline.tenant_id, - datasource_type=DatasourceProviderType(datasource_type), - ) - credentials = datasource_providers.get_datasource_credentials( - tenant_id=pipeline.tenant_id, - provider=datasource_node_data.get("provider_name"), - plugin_id=datasource_node_data.get("plugin_id"), - credential_id=credential_id, - ) - if credentials: - datasource_runtime.runtime.credentials = credentials - match datasource_type: - case DatasourceProviderType.ONLINE_DOCUMENT: - datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime) - online_document_result: Generator[DatasourceMessage] = ( - datasource_runtime.get_online_document_page_content( - user_id=account.id, - datasource_parameters=GetOnlineDocumentPageContentRequest( - workspace_id=user_inputs.get("workspace_id", ""), - page_id=user_inputs.get("page_id", ""), - type=user_inputs.get("type", ""), - ), - provider_type=datasource_type, - ) - ) - try: - variables: dict[str, Any] = {} - for online_document_message in online_document_result: - if online_document_message.type == DatasourceMessage.MessageType.VARIABLE: - assert isinstance(online_document_message.message, DatasourceMessage.VariableMessage) - variable_name = online_document_message.message.variable_name - variable_value = online_document_message.message.variable_value - if online_document_message.message.stream: - if not isinstance(variable_value, str): - raise ValueError("When 'stream' is True, 'variable_value' must be a string.") - if variable_name not in variables: - variables[variable_name] = "" - variables[variable_name] += variable_value - else: - variables[variable_name] = variable_value - return variables - except Exception as e: - logger.exception("Error during get online document content.") - raise RuntimeError(str(e)) from e - # TODO Online Drive - case _: - raise ValueError(f"Unsupported datasource provider: {datasource_runtime.datasource_provider_type}") - except Exception as e: - logger.exception("Error in run_datasource_node_preview.") - raise RuntimeError(str(e)) from e - - def run_free_workflow_node( - self, node_data: dict, tenant_id: str, user_id: str, node_id: str, user_inputs: dict[str, Any] - ) -> WorkflowNodeExecution: - """ - Run draft workflow node - """ - # run draft workflow node - start_at = time.perf_counter() - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.run_free_node( - node_id=node_id, - node_data=node_data, - tenant_id=tenant_id, - user_id=user_id, - user_inputs=user_inputs, - ), - start_at=start_at, - tenant_id=tenant_id, - node_id=node_id, - ) - - return workflow_node_execution - - def _handle_node_run_result( - self, - getter: Callable[ - [], - tuple[Node, Generator[NodeEvent | ContainerAwaitRequest]], - ], - start_at: float, - tenant_id: str, - node_id: str, - ) -> WorkflowNodeExecution: - """ - Handle node run result - - :param getter: Callable[[], tuple[BaseNode, Generator[RunEvent | InNodeEvent, None, None]]] - :param start_at: float - :param tenant_id: str - :param node_id: str - """ - try: - node_instance, generator = getter() - - node_run_result: NodeRunResult | None = None - for event in generator: - if isinstance(event, (NodeRunSucceededEvent, NodeRunFailedEvent)): - node_run_result = event.node_run_result - if node_run_result: - # sign output files - node_run_result.outputs = WorkflowEntry.handle_special_values(node_run_result.outputs) or {} - break - - if not node_run_result: - raise ValueError("Node run failed with no run result") - # single step debug mode error handling return - if node_run_result.status == WorkflowNodeExecutionStatus.FAILED and node_instance.error_strategy: - node_error_args: dict[str, Any] = { - "status": WorkflowNodeExecutionStatus.EXCEPTION, - "error": node_run_result.error, - "inputs": node_run_result.inputs, - "metadata": {"error_strategy": node_instance.error_strategy}, - } - if node_instance.error_strategy is ErrorStrategy.DEFAULT_VALUE: - node_run_result = NodeRunResult( - **node_error_args, - outputs={ - **node_instance.default_value_dict, - "error_message": node_run_result.error, - "error_type": node_run_result.error_type, - }, - ) - else: - node_run_result = NodeRunResult( - **node_error_args, - outputs={ - "error_message": node_run_result.error, - "error_type": node_run_result.error_type, - }, - ) - run_succeeded = node_run_result.status in ( - WorkflowNodeExecutionStatus.SUCCEEDED, - WorkflowNodeExecutionStatus.EXCEPTION, - ) - error = node_run_result.error if not run_succeeded else None - except WorkflowNodeRunFailedError as e: - node_instance = e._node # type: ignore - run_succeeded = False - node_run_result = None - error = e._error # type: ignore - - workflow_node_execution = WorkflowNodeExecution( - id=str(uuid4()), - workflow_id=node_instance.workflow_id, - index=1, - node_id=node_id, - node_type=node_instance.node_type, - title=node_instance.title, - elapsed_time=time.perf_counter() - start_at, - finished_at=datetime.now(UTC).replace(tzinfo=None), - created_at=datetime.now(UTC).replace(tzinfo=None), - ) - if run_succeeded and node_run_result: - # create workflow node execution - inputs = WorkflowEntry.handle_special_values(node_run_result.inputs) if node_run_result.inputs else None - process_data = ( - WorkflowEntry.handle_special_values(node_run_result.process_data) - if node_run_result.process_data - else None - ) - outputs = WorkflowEntry.handle_special_values(node_run_result.outputs) if node_run_result.outputs else None - - workflow_node_execution.inputs = inputs - workflow_node_execution.process_data = process_data - workflow_node_execution.outputs = outputs - workflow_node_execution.metadata = node_run_result.metadata - if node_run_result.status == WorkflowNodeExecutionStatus.SUCCEEDED: - workflow_node_execution.status = WorkflowNodeExecutionStatus.SUCCEEDED - elif node_run_result.status == WorkflowNodeExecutionStatus.EXCEPTION: - workflow_node_execution.status = WorkflowNodeExecutionStatus.EXCEPTION - workflow_node_execution.error = node_run_result.error - else: - # create workflow node execution - workflow_node_execution.status = WorkflowNodeExecutionStatus.FAILED - workflow_node_execution.error = error - # update document status - variable_pool = node_instance.runtime_state.variable_pool - invoke_from = get_system_segment(variable_pool, SystemVariableKey.INVOKE_FROM) - if invoke_from: - if invoke_from.value == InvokeFrom.PUBLISHED_PIPELINE: - document_id = get_system_segment(variable_pool, SystemVariableKey.DOCUMENT_ID) - dataset_id = get_system_segment(variable_pool, SystemVariableKey.DATASET_ID) - pipeline_id = get_system_segment(variable_pool, SystemVariableKey.APP_ID) - if document_id and dataset_id and pipeline_id: - dataset = self._session.scalar( - select(Dataset) - .where( - Dataset.id == dataset_id.value, - Dataset.tenant_id == tenant_id, - Dataset.pipeline_id == pipeline_id.value, - ) - .limit(1) - ) - if dataset: - dataset_ref = DatasetRef(tenant_id=dataset.tenant_id, dataset_id=dataset.id) - document_ref = dataset_ref.document(document_id.value) - # dataset_service imports RagPipelineService; importing here avoids the cycle. - from services.knowledge.dataset_service import DocumentService - - document = next( - iter( - DocumentService.get_documents_by_ids( - document_ref.dataset, [document_ref.document_id], self._session - ) - ), - None, - ) - if document: - document.indexing_status = IndexingStatus.ERROR - document.error = error - self._session.add(document) - self._session.commit() - - return workflow_node_execution - def update_workflow( self, *, @@ -1096,7 +328,7 @@ def update_workflow( account_id: str, data: dict[str, Any], workflow_ref: WorkflowRef, - ) -> Workflow | None: + ) -> WorkflowRecord | None: """ Update workflow attributes @@ -1106,26 +338,10 @@ def update_workflow( :param workflow_ref: Owner-bound workflow reference :return: Updated workflow or None if not found """ - stmt = select(Workflow).where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, + workflow = WorkflowDefinitionStore.update_workflow( + session=session, account_id=account_id, data=data, workflow_ref=workflow_ref ) - workflow = session.scalar(stmt) - - if not workflow: - return None - - allowed_fields = ["marked_name", "marked_comment"] - - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account_id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - - return workflow + return workflow_record(workflow, session) if workflow is not None else None def get_first_step_parameters(self, pipeline: Pipeline, node_id: str, is_draft: bool = False) -> list[dict]: """ @@ -1258,10 +474,7 @@ def get_rag_pipeline_workflow_run(self, pipeline: Pipeline, run_id: str) -> Work ) def get_rag_pipeline_workflow_run_node_executions( - self, - pipeline: Pipeline, - run_id: str, - user: Account | EndUser, + self, pipeline: Pipeline, run_id: str, user: Account | EndUser, *, tool_providers: ToolProviders ) -> list[WorkflowNodeExecutionTrace]: """ Get workflow run node execution list @@ -1280,7 +493,7 @@ def get_rag_pipeline_workflow_run_node_executions( workflow_run_id=run_id, ) return assemble_workflow_node_execution_traces( - node_executions, self._node_execution_service_repo, session=self._session + node_executions, self._node_execution_service_repo, session=self._session, tool_providers=tool_providers ) @staticmethod @@ -1349,18 +562,6 @@ def publish_customized_pipeline_template( session.add(pipeline_customized_template) session.commit() - def is_workflow_exist(self, pipeline: Pipeline) -> bool: - return ( - self._session.scalar( - select(func.count(Workflow.id)).where( - Workflow.tenant_id == pipeline.tenant_id, - Workflow.app_id == pipeline.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - ) - or 0 - ) > 0 - def get_node_last_run( self, pipeline: Pipeline, workflow: Workflow, node_id: str ) -> WorkflowNodeExecutionModel | None: @@ -1372,103 +573,15 @@ def get_node_last_run( ) return node_exec - def set_datasource_variables(self, pipeline: Pipeline, args: dict[str, Any], current_user: Account): - """ - Set datasource variables - """ - - # fetch draft workflow by app_model - draft_workflow = self.get_draft_workflow(pipeline=pipeline) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - # run draft workflow node - start_at = time.perf_counter() - node_id = args.get("start_node_id") - if not node_id: - raise ValueError("Node id is required") - node_config = draft_workflow.get_node_config_by_id(node_id) - - eclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - if eclosing_node_type_and_id: - _, enclosing_node_id = eclosing_node_type_and_id - else: - enclosing_node_id = None - - system_inputs = build_system_variables( - datasource_type=args.get("datasource_type", "online_document"), - datasource_info=args.get("datasource_info", {}), - ) - - workflow_node_execution = self._handle_node_run_result( - getter=lambda: WorkflowEntry.single_step_run( - execution_driver=WorkflowRunAgg.run, - workflow=draft_workflow, - node_id=node_id, - user_inputs={}, - user_id=current_user.id, - variable_pool=_build_seeded_variable_pool( - build_bootstrap_variables( - system_variables=system_inputs, - rag_pipeline_variables=(), - ) - ), - variable_loader=DraftVarLoader( - engine=db.engine, - app_id=pipeline.id, - tenant_id=pipeline.tenant_id, - user_id=current_user.id, - ), - ), - start_at=start_at, - tenant_id=pipeline.tenant_id, - node_id=node_id, - ) - workflow_node_execution.workflow_id = draft_workflow.id - - # Create repository and save the node execution - repository = SQLAlchemyWorkflowNodeExecutionRepository( - session_factory=db.engine, - tenant_id=pipeline.tenant_id, - user=current_user, - app_id=pipeline.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, - ) - repository.save(workflow_node_execution) - - # Convert node_execution to WorkflowNodeExecution after save - workflow_node_execution_db_model = repository._to_db_model(workflow_node_execution) # type: ignore - - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=pipeline.tenant_id, - app_id=pipeline.id, - node_id=workflow_node_execution_db_model.node_id, - node_type=workflow_node_execution_db_model.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=workflow_node_execution.id, - user=current_user, - ) - draft_var_saver.save( - process_data=workflow_node_execution.process_data, - outputs=workflow_node_execution.outputs, - ) - enqueue_draft_node_execution_trace( - execution=workflow_node_execution_db_model, - outputs=workflow_node_execution.outputs, - workflow_execution_id=None, - user_id=current_user.id, - ) - return workflow_node_execution_db_model - def _fetch_recommended_plugin_manifests(self, plugin_ids: list[str]) -> list[Any]: if not dify_config.MARKETPLACE_ENABLED: logger.info("Marketplace disabled; recommended-plugins list empty") return [] return marketplace.batch_fetch_plugin_by_ids(plugin_ids) - def get_recommended_plugins(self, type: str, current_user: Account, current_tenant_id: str) -> dict[str, Any]: + def get_recommended_plugins( + self, type: str, current_user: Account, current_tenant_id: str, *, tool_providers: ToolProviders + ) -> dict[str, Any]: # Query active recommended plugins stmt = select(PipelineRecommendedPlugin).where(PipelineRecommendedPlugin.active.is_(True)) if type and type != "all": @@ -1487,8 +600,7 @@ def get_recommended_plugins(self, type: str, current_user: Account, current_tena # Batch fetch plugin manifests plugin_ids = [plugin.plugin_id for plugin in pipeline_recommended_plugins] providers = BuiltinToolManageService.list_builtin_tools( - user_id=current_user.id, - tenant_id=current_tenant_id, + user_id=current_user.id, tenant_id=current_tenant_id, tool_providers=tool_providers ) providers_map = {provider.plugin_id: provider.to_dict() for provider in providers} @@ -1530,9 +642,8 @@ def retry_error_document( if not workflow: raise ValueError("Workflow not found") generator.generate( - session=self._session, pipeline=pipeline, - workflow=workflow, + workflow=workflow_snapshot(workflow), user=user, args={ "inputs": document_pipeline_execution_log.input_data, diff --git a/api/services/snippet_dsl_service.py b/api/services/snippet_dsl_service.py index d622b5c23f1c8b..47f9f6e4d37cc1 100644 --- a/api/services/snippet_dsl_service.py +++ b/api/services/snippet_dsl_service.py @@ -16,12 +16,12 @@ from extensions.ext_redis import redis_client from graphon.enums import BuiltinNodeTypes from graphon.model_runtime.utils.encoders import jsonable_encoder +from machinery.context import RequestContext from models import Account from models.snippet import CustomizedSnippet, SnippetType from models.workflow import Workflow from services.agent.dsl_service import AgentDslService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.dsl_content import DSL_MAX_SIZE, dsl_content_size from services.dsl_version import check_version_compatibility from services.entities.dsl_entities import ( @@ -32,7 +32,10 @@ PendingImportOwner, ) from services.plugin.dependencies_analysis import DependenciesAnalysisService -from services.snippet_service import SNIPPET_FORBIDDEN_NODE_TYPES, SnippetService +from services.snippet_service import SnippetService +from services.workflow.contracts import DraftImportResult, DraftSyncCommand, WorkflowOwner, WorkflowSnapshot +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.snippet_policy import SNIPPET_FORBIDDEN_NODE_TYPES logger = logging.getLogger(__name__) @@ -71,13 +74,18 @@ class CheckDependenciesPendingData(BaseModel): class SnippetDslService: - def __init__(self, session: Session): + def __init__( + self, + session: Session, + *, + snippets: SnippetService, + drafts: WorkflowDraftService[WorkflowAgentBindingStore], + ): self._session = session + self._snippets = snippets + self._drafts = drafts self._warnings: list[DslImportWarning] = [] - def _snippet_service(self) -> SnippetService: - return SnippetService(session=self._session) - def import_snippet( self, *, @@ -383,7 +391,7 @@ def check_dependencies(self, snippet: CustomizedSnippet) -> CheckDependenciesRes """ Check dependencies for a snippet """ - snippet_service = self._snippet_service() + snippet_service = self._snippets workflow = snippet_service.get_draft_workflow(snippet=snippet) if not workflow: return CheckDependenciesResult(leaked_dependencies=[]) @@ -448,7 +456,6 @@ def _create_or_update_snippet( self._session.flush() # Create or update draft workflow - retirement_candidates: set[str] = set() if workflow_data: graph = workflow_data.get("graph", {}) raw_agent_packages = data.get("agent_packages") or {} @@ -456,49 +463,32 @@ def _create_or_update_snippet( raise ValueError("agent_packages must be a mapping") graph_for_sync = AgentDslService.graph_without_package_bindings(graph) if raw_agent_packages else graph - snippet_service = self._snippet_service() - # Get existing workflow hash if exists - existing_workflow = snippet_service.get_draft_workflow(snippet=snippet) - unique_hash = existing_workflow.unique_hash if existing_workflow else None - - draft_workflow = snippet_service.sync_draft_workflow( - snippet=snippet, - graph=graph_for_sync, - unique_hash=unique_hash, - account=account, - input_fields=input_fields, - sync_agent_bindings=False, - ) - if raw_agent_packages: - _, warnings, retirement_candidates = AgentDslService(self._session).import_workflow_packages( - workflow=draft_workflow, - portable_graph=graph, - raw_packages=raw_agent_packages, - account=account, + def materialize(workflow: WorkflowSnapshot) -> DraftImportResult: + imported_graph, warnings, retired = AgentDslService(self._session).import_workflow_packages( + workflow=workflow, portable_graph=graph, raw_packages=raw_agent_packages, account=account ) self._warnings.extend(warnings) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) - else: - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=self._session, - draft_workflow=draft_workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=self._session, - draft_workflow=draft_workflow, - ) + return DraftImportResult(imported_graph, retired) + + self._drafts.sync( + RequestContext(str(uuid.uuid4()), None, account.id, snippet.tenant_id), + WorkflowOwner(snippet.id, "snippet"), + DraftSyncCommand( + graph=graph_for_sync, + features={}, + unique_hash=None, + check_hash=False, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=input_fields, + ), + materialize=materialize if raw_agent_packages else None, + ) self._session.commit() - if workflow_data: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) return snippet def export_snippet_dsl( @@ -511,7 +501,7 @@ def export_snippet_dsl( :param workflow_id: Optional published workflow version to export; defaults to the draft workflow :return: YAML string """ - snippet_service = self._snippet_service() + snippet_service = self._snippets workflow = ( snippet_service.get_published_workflow_by_id(snippet=snippet, workflow_id=workflow_id) if workflow_id diff --git a/api/services/snippet_generate_service.py b/api/services/snippet_generate_service.py index bab575c0a990db..0480638c64b572 100644 --- a/api/services/snippet_generate_service.py +++ b/api/services/snippet_generate_service.py @@ -21,12 +21,12 @@ import json import logging from collections.abc import Generator, Mapping, Sequence -from typing import Any, cast +from dataclasses import replace +from typing import Any, Protocol, cast -from sqlalchemy.orm import Session, make_transient, sessionmaker +from sqlalchemy.orm import make_transient from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.file_access import DatabaseFileAccessController from core.workflow.snippet_start import SNIPPET_VIRTUAL_START_NODE_ID @@ -37,15 +37,23 @@ from models.model import App, AppMode, EndUser from models.snippet import CustomizedSnippet from models.workflow import Workflow, WorkflowNodeExecutionModel -from services.snippet_service import SnippetService +from repositories.workflow.definition_repository import workflow_snapshot +from services.app.generation.response import convert_to_event_stream +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_execution_identity import build_workflow_execution_identity -from services.workflow_run_agg import WorkflowRunAgg from services.workflow_service import WorkflowService logger = logging.getLogger(__name__) _file_access_controller = DatabaseFileAccessController() +class SnippetWorkflowReader(Protocol): + def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: ... + + class _SnippetAsApp: """ Minimal adapter that wraps a CustomizedSnippet to satisfy the App-like @@ -86,6 +94,19 @@ class SnippetGenerateService: # Specific ID for the injected virtual Start node so it can be recognised _VIRTUAL_START_NODE_ID = SNIPPET_VIRTUAL_START_NODE_ID + def __init__( + self, + *, + snippets: SnippetWorkflowReader, + variables: WorkflowExecutionVariables, + runtime: WorkflowRuntime, + workflows: WorkflowService, + ) -> None: + self._snippets = snippets + self._variables = variables + self._runtime = runtime + self._workflows = workflows + @classmethod def _is_virtual_start_event(cls, message: Mapping[str, Any] | str) -> bool: """ @@ -130,15 +151,13 @@ def _stream() -> Generator[Mapping[str, Any] | str]: return _stream() - @classmethod def generate( - cls, + self, snippet: CustomizedSnippet, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, streaming: bool = True, - session_maker: sessionmaker[Session] | None = None, ) -> Mapping[str, Any] | Generator[str]: """ Run a snippet's draft workflow. @@ -159,18 +178,23 @@ def generate( :return: Blocking response mapping or SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) # Inject a virtual Start node when the graph doesn't have one. - workflow = cls._ensure_start_node(workflow, snippet) + workflow = self._ensure_start_node(workflow, snippet) # Adapt snippet to App-like interface for WorkflowAppGenerator app_proxy = cast(App, _SnippetAsApp(snippet)) - response = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate( + response = WorkflowAppGenerator( + runtime=self._runtime, + execution_driver=self._runtime.execution_driver, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).generate( app_model=app_proxy, workflow=workflow, user=user, @@ -181,67 +205,16 @@ def generate( call_depth=0, ) - return WorkflowAppGenerator.convert_to_event_stream(cls._filter_virtual_start_events(response)) + return convert_to_event_stream(self._filter_virtual_start_events(response)) @classmethod - def run_published( - cls, - snippet: CustomizedSnippet, - user: Account | EndUser, - args: Mapping[str, Any], - invoke_from: InvokeFrom, - session_maker: sessionmaker[Session] | None = None, - ) -> Mapping[str, Any]: - """ - Run a snippet's published workflow in non-streaming (blocking) mode. - - Similar to :meth:`generate` but targets the published workflow instead - of the draft, and returns the raw blocking response without SSE - wrapping. Designed for programmatic callers that need direct workflow outputs. - - :param snippet: CustomizedSnippet instance (must be published) - :param user: Account or EndUser initiating the run - :param args: Workflow inputs (must include "inputs" key) - :param invoke_from: Source of invocation - :return: Blocking response mapping with workflow outputs - :raises ValueError: If the snippet has no published workflow - """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_published_workflow(snippet) - if not workflow: - raise ValueError("No published workflow found for snippet") - - # Inject a virtual Start node when the graph doesn't have one. - workflow = cls._ensure_start_node(workflow, snippet) - - app_proxy = cast(App, _SnippetAsApp(snippet)) - - response = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate( - app_model=app_proxy, - workflow=workflow, - user=user, - execution_identity=build_workflow_execution_identity(user), - args=args, - invoke_from=invoke_from, - streaming=False, - call_depth=0, - ) - return response - - @classmethod - def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: - """Public wrapper for worker-thread start-node injection.""" - return cls._ensure_start_node(workflow, snippet) - - @classmethod - def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: + def _ensure_start_node(cls, workflow: WorkflowSnapshot, snippet: CustomizedSnippet) -> WorkflowSnapshot: """ Return *workflow* with a Start node. If the graph already contains a Start node, the original workflow is returned unchanged. Otherwise a virtual Start node is injected and the - workflow object is detached from the SQLAlchemy session so the in-memory - change is never flushed to the database. + returned snapshot contains the in-memory graph used for this execution. """ graph_dict = workflow.graph_dict nodes: list[dict[str, Any]] = graph_dict.get("nodes", []) @@ -255,10 +228,18 @@ def _ensure_start_node(cls, workflow: Workflow, snippet: CustomizedSnippet) -> W input_fields=snippet.input_fields_list, ) - # Detach from session to prevent accidental DB persistence of the - # modified graph. All attributes remain accessible for read. + return replace(workflow, graph=json.dumps(modified_graph)) + + @classmethod + def ensure_start_node_for_worker(cls, workflow: Workflow, snippet: CustomizedSnippet) -> Workflow: + """Inject the virtual Start node after a worker reloads a snippet workflow.""" + snapshot = workflow_snapshot(workflow) + updated_snapshot = cls._ensure_start_node(snapshot, snippet) + if updated_snapshot.graph == snapshot.graph: + return workflow + make_transient(workflow) - workflow.graph = json.dumps(modified_graph) + workflow.graph = updated_snapshot.graph return workflow @classmethod @@ -334,16 +315,14 @@ def _inject_virtual_start_node( "edges": [*edges, *new_edges], } - @classmethod def run_draft_node( - cls, + self, snippet: CustomizedSnippet, node_id: str, user_inputs: Mapping[str, Any], account: Account, query: str = "", files: Sequence[File] | None = None, - session_maker: sessionmaker[Session] | None = None, ) -> WorkflowNodeExecutionModel: """ Run a single node in a snippet's draft workflow (single-step debugging). @@ -360,15 +339,15 @@ def run_draft_node( :return: WorkflowNodeExecutionModel with execution results :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - draft_workflow = snippet_service.get_draft_workflow(snippet=snippet) - if not draft_workflow: + draft_workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not draft_workflow_model: raise ValueError("Workflow not initialized") + draft_workflow = workflow_snapshot(draft_workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) - workflow_service = WorkflowService() - return workflow_service.run_draft_workflow_node( + return self._workflows.run_draft_workflow_node( + variables=self._variables, app_model=app_proxy, draft_workflow=draft_workflow, node_id=node_id, @@ -378,16 +357,13 @@ def run_draft_node( files=files, ) - @classmethod def generate_single_iteration( - cls, + self, snippet: CustomizedSnippet, user: Account | EndUser, node_id: str, args: Mapping[str, Any], streaming: bool = True, - *, - session_maker: sessionmaker[Session], ) -> Mapping[str, Any] | Generator[str]: """ Run a single iteration node in a snippet's draft workflow. @@ -401,41 +377,40 @@ def generate_single_iteration( :param node_id: ID of the iteration node to run :param args: Dict containing 'inputs' key with iteration input data :param streaming: Whether to stream the response (should be True) - :param session_maker: Factory for the synchronous database work before worker startup :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) - with session_maker() as session: - return WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_iteration_generate( - app_model=app_proxy, - workflow=workflow, - node_id=node_id, - user=user, - execution_identity=build_workflow_execution_identity(user), - args=args, - streaming=streaming, - session=session, - ) + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=self._runtime, + execution_driver=self._runtime.execution_driver, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).single_iteration_generate( + app_model=app_proxy, + workflow=workflow, + node_id=node_id, + user=user, + execution_identity=build_workflow_execution_identity(user), + args=args, + streaming=streaming, ) + ) - @classmethod def generate_single_loop( - cls, + self, snippet: CustomizedSnippet, user: Account | EndUser, node_id: str, - args: Any, + args: Mapping[str, Any], streaming: bool = True, - *, - session_maker: sessionmaker[Session], ) -> Mapping[str, Any] | Generator[str]: """ Run a single loop node in a snippet's draft workflow. @@ -447,32 +422,34 @@ def generate_single_loop( :param snippet: CustomizedSnippet instance :param user: Account or EndUser initiating the run :param node_id: ID of the loop node to run - :param args: Pydantic model with 'inputs' attribute containing loop input data + :param args: Mapping with 'inputs' containing loop input data :param streaming: Whether to stream the response (should be True) - :param session_maker: Factory for the synchronous database work before worker startup :return: SSE streaming generator :raises ValueError: If the snippet has no draft workflow """ - snippet_service = SnippetService(session_maker) - workflow = snippet_service.get_draft_workflow(snippet=snippet) - if not workflow: + workflow_model = self._snippets.get_draft_workflow(snippet=snippet) + if not workflow_model: raise ValueError("Workflow not initialized") + workflow = workflow_snapshot(workflow_model) app_proxy = cast(App, _SnippetAsApp(snippet)) - with session_maker() as session: - return WorkflowAppGenerator.convert_to_event_stream( - WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).single_loop_generate( - app_model=app_proxy, - workflow=workflow, - node_id=node_id, - user=user, - execution_identity=build_workflow_execution_identity(user), - args=args, # type: ignore[arg-type] - streaming=streaming, - session=session, - ) + return convert_to_event_stream( + WorkflowAppGenerator( + runtime=self._runtime, + execution_driver=self._runtime.execution_driver, + draft_variable_loader=self._variables.workflow_loader, + draft_variable_saver=self._variables.saver_factory, + ).single_loop_generate( + app_model=app_proxy, + workflow=workflow, + node_id=node_id, + user=user, + execution_identity=build_workflow_execution_identity(user), + args=args, + streaming=streaming, ) + ) @staticmethod def parse_files(workflow: Workflow, files: list[dict] | None = None) -> Sequence[File]: diff --git a/api/services/snippet_service.py b/api/services/snippet_service.py index 43fb7359db1305..cbe354c654ef0a 100644 --- a/api/services/snippet_service.py +++ b/api/services/snippet_service.py @@ -1,7 +1,8 @@ import json import logging -from collections.abc import Generator, Mapping, Sequence +from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import contextmanager +from dataclasses import replace from datetime import UTC, datetime from typing import Any @@ -9,20 +10,13 @@ from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.workflow.node_factory import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING from enums import DeploymentEdition -from graphon.enums import BuiltinNodeTypes, NodeType +from enums.agent import WorkflowAgentBindingType +from graphon.enums import NodeType from libs.helper import escape_like_pattern from libs.infinite_scroll_pagination import InfiniteScrollPagination from models import Account, TagBinding -from models.agent import ( - WORKFLOW_ONLY_AGENT_SOURCES, - Agent, - AgentScope, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import WORKFLOW_ONLY_AGENT_SOURCES, Agent, AgentScope, AgentStatus, WorkflowAgentNodeBinding from models.enums import WorkflowRunTriggeredFrom from models.model import UploadFile from models.snippet import CustomizedSnippet, SnippetType @@ -36,30 +30,29 @@ WorkflowKind, WorkflowNodeExecutionModel, WorkflowRun, - WorkflowType, ) +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope from repositories.factory import DifyAPIRepositoryFactory -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from repositories.workflow.definition_repository import WorkflowDefinitionStore, workflow_from_snapshot, workflow_record +from repositories.workflow.snippet_publication_repository import SnippetPublicationRepository +from services.agent.retirement_service import WorkflowAgentRetirementService +from services.agent.workflow_contracts import AgentSkillReader +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.errors.app import IsDraftWorkflowError from services.tag_application_service import TagTargetQuery +from services.tools.provider_queries import ToolProviders +from services.workflow.contracts import WorkflowRecord +from services.workflow.execution.adapters.node_factory import LATEST_VERSION, NODE_TYPE_CLASSES_MAPPING +from services.workflow.snippet_policy import validate_snippet_graph_forbidden_nodes from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, ) -from services.workflow_restore import apply_published_workflow_snapshot_to_draft +from services.workflow_ref_service import WorkflowRef logger = logging.getLogger(__name__) -# Node types not allowed in snippet workflows (sync, publish, DSL import). -SNIPPET_FORBIDDEN_NODE_TYPES: frozenset[str] = frozenset( - { - BuiltinNodeTypes.START, - BuiltinNodeTypes.HUMAN_INPUT, - BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, - } -) - class SnippetService: """Service for managing customized snippets.""" @@ -68,6 +61,9 @@ def __init__( self, session_maker: sessionmaker[Session] | Session | None = None, session: Session | None = None, + *, + agent_bindings: Callable[[Session], WorkflowAgentPublishService], + skills: AgentSkillReader, ): """Initialize SnippetService with repository dependencies.""" if isinstance(session_maker, Session): @@ -78,7 +74,10 @@ def __init__( if session_maker is None: raise ValueError("SnippetService requires a session or session_maker.") self._session = session + self._agent_bindings = agent_bindings self._session_maker = session_maker + self._publication = SnippetPublicationRepository(session_maker) + self._skills = skills self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) @@ -178,30 +177,6 @@ def _delete_archived_workflow_run_files(*, snippet: CustomizedSnippet) -> None: except Exception: logger.exception("Failed to delete snippet archive file %s", key) - @staticmethod - def validate_snippet_graph_forbidden_nodes(graph: Mapping[str, Any]) -> None: - """Reject graphs that contain node types not allowed in snippets.""" - nodes = graph.get("nodes") or [] - disallowed: list[tuple[str, str]] = [] - for node in nodes: - if not isinstance(node, dict): - continue - node_data = node.get("data") or {} - node_type = node_data.get("type") - if not isinstance(node_type, str): - continue - if node_type in SNIPPET_FORBIDDEN_NODE_TYPES: - node_id = node.get("id") - disallowed.append((str(node_id) if node_id is not None else "?", node_type)) - if not disallowed: - return - detail = ", ".join(f"{nid}:{t}" for nid, t in disallowed) - raise ValueError( - f"Snippet workflow cannot contain start, human-input, or knowledge-retrieval nodes. Found: {detail}" - ) - - # --- CRUD Operations --- - def get_snippets( self, *, @@ -297,6 +272,7 @@ def create_snippet( icon_info: dict | None, input_fields: list[dict] | None, account: Account, + graph: dict[str, Any] | None = None, ) -> CustomizedSnippet: """ Create a new snippet. @@ -310,6 +286,8 @@ def create_snippet( :param account: Creator account :return: Created CustomizedSnippet """ + if graph is not None: + validate_snippet_graph_forbidden_nodes(graph) snippet = CustomizedSnippet( tenant_id=tenant_id, name=name, @@ -403,7 +381,9 @@ def delete_snippet( tenant_id = snippet.tenant_id def collect_agent_resources(_session: Session) -> None: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService( + WorkflowAgentRetirementRepository(sessionmaker(bind=session.get_bind(), expire_on_commit=False)) + ).retire_unowned( tenant_id=tenant_id, agent_ids=candidate_agent_ids, account_id=account_id, @@ -495,13 +475,20 @@ def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: :return: Draft Workflow or None """ with self._session_scope() as session: - stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version == "draft", + return WorkflowDefinitionStore.get_draft_workflow(snippet, session=session, kind=WorkflowKind.SNIPPET.value) + + def get_draft_workflow_record(self, snippet: CustomizedSnippet) -> WorkflowRecord | None: + """Materialize editor data, including Agent bindings, before serialization.""" + with self._session_scope() as session: + workflow = WorkflowDefinitionStore.get_draft_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value ) - return session.scalar(stmt) + if workflow is None: + return None + graph = self._agent_bindings(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow) + ) + return replace(workflow_record(workflow, session), graph=graph, conversation_variables=[]) def get_published_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: """ @@ -514,13 +501,16 @@ def get_published_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: return None with self._session_scope() as session: - stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.id == snippet.workflow_id, + return WorkflowDefinitionStore.get_published_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value ) - return session.scalar(stmt) + + def get_published_workflow_record(self, snippet: CustomizedSnippet) -> WorkflowRecord | None: + with self._session_scope() as session: + workflow = WorkflowDefinitionStore.get_published_workflow( + snippet, session=session, kind=WorkflowKind.SNIPPET.value + ) + return workflow_record(workflow, session) if workflow is not None else None def get_published_workflow_by_id(self, snippet: CustomizedSnippet, workflow_id: str) -> Workflow | None: """ @@ -545,217 +535,28 @@ def get_published_workflow_by_id(self, snippet: CustomizedSnippet, workflow_id: raise IsDraftWorkflowError("source workflow must be published") return workflow - def sync_draft_workflow( - self, - *, - snippet: CustomizedSnippet, - graph: dict, - unique_hash: str | None, - account: Account, - input_fields: list[dict] | None = None, - sync_agent_bindings: bool = True, - ) -> Workflow: - """ - Sync draft workflow for snippet. - - Snippet workflows do not persist environment variables (always empty) or - conversation variables (always empty). - - :param snippet: CustomizedSnippet instance - :param graph: Workflow graph configuration - :param unique_hash: Hash for conflict detection - :param account: Account making the change - :param input_fields: Input fields for snippet - :return: Synced Workflow - :raises WorkflowHashNotEqualError: If hash mismatch - """ - SnippetService.validate_snippet_graph_forbidden_nodes(graph) - - workflow = self.get_draft_workflow(snippet=snippet) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # Create draft workflow if not found - if not workflow: - workflow = Workflow( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - _features="{}", - type=WorkflowType.WORKFLOW, - kind=WorkflowKind.SNIPPET, - version="draft", - graph=json.dumps(graph), - created_by=account.id, - _environment_variables=[], - _conversation_variables=[], - ) - else: - # Update existing draft workflow - workflow.graph = json.dumps(graph) - workflow.type = WorkflowType.WORKFLOW - workflow.kind = WorkflowKind.SNIPPET - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - workflow.environment_variables = [] - workflow.conversation_variables = [] - - # Update snippet's input_fields if provided - if input_fields is not None: - snippet.input_fields = json.dumps(input_fields) - snippet.updated_by = account.id - snippet.updated_at = datetime.now(UTC).replace(tzinfo=None) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - retirement_candidates: set[str] = set() - with self._session_scope() as session: - session.add(workflow) - session.add(snippet) - if sync_agent_bindings: - session.flush() - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=session, - draft_workflow=workflow, - ) - self._commit_if_owned(session) - if self._session is None: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - return workflow - - def restore_published_workflow_to_draft( - self, - *, - snippet: CustomizedSnippet, - workflow_id: str, - account: Account, - ) -> Workflow: - """ - Restore a published snippet workflow snapshot into the draft workflow. - - :param snippet: CustomizedSnippet instance - :param workflow_id: Published workflow ID - :param account: Account making the change - :return: Restored draft Workflow - :raises WorkflowNotFoundError: If the source workflow does not exist - :raises IsDraftWorkflowError: If the source workflow is a draft - :raises ValueError: If the restored graph is invalid for snippets - """ - source_workflow = self.get_published_workflow_by_id(snippet=snippet, workflow_id=workflow_id) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - SnippetService.validate_snippet_graph_forbidden_nodes(source_workflow.graph_dict) - - draft_workflow = self.get_draft_workflow(snippet=snippet) - draft_workflow, _is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=lambda: datetime.now(UTC).replace(tzinfo=None), - ) - - with self._session_scope() as session: - session.add(draft_workflow) - session.flush() - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - retirement_candidates = WorkflowAgentPublishService.restore_agent_node_bindings_to_draft( - session=session, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account_id=account.id, - ) - self._commit_if_owned(session) - if self._session is None: - WorkflowAgentRetirementService.retire_unowned( - tenant_id=snippet.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - snippet: CustomizedSnippet, - account: Account, - ) -> Workflow: - """ - Publish the draft workflow as a new version. - - :param session: Database session - :param snippet: CustomizedSnippet instance - :param account: Account making the change - :return: Published Workflow - :raises ValueError: If no draft workflow exists - """ - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version == "draft", - ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") - - SnippetService.validate_snippet_graph_forbidden_nodes(draft_workflow.graph_dict) + def publish_workflow(self, *, snippet: CustomizedSnippet, account: Account) -> Workflow: + """Prepare external dependencies without a transaction, then recheck and publish.""" + with self._publication.draft(tenant_id=snippet.tenant_id, snippet_id=snippet.id) as (source, bindings): + agents = WorkflowAgentPublishService(repository=bindings).publication_state(draft_workflow=source) + draft = workflow_from_snapshot(source) + validate_snippet_graph_forbidden_nodes(draft.graph_dict) from core.workflow.llm_environment_variable import validate_llm_environment_model_references validate_llm_environment_model_references( - graph=draft_workflow.graph_dict, - environment_variables=draft_workflow.environment_variables, + graph=draft.graph_dict, + environment_variables=draft.environment_variables, ) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=draft_workflow, - ) - - # Create new published workflow - workflow = Workflow.new( - tenant_id=snippet.tenant_id, - app_id=snippet.id, - type=WorkflowType.WORKFLOW.value, - version=str(datetime.now(UTC).replace(tzinfo=None)), - graph=draft_workflow.graph, - features=draft_workflow.features, - created_by=account.id, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - kind=WorkflowKind.SNIPPET.value, - ) - session.add(workflow) - WorkflowAgentPublishService.copy_agent_node_bindings_to_published( - session=session, - draft_workflow=draft_workflow, - published_workflow=workflow, - ) - - # Update snippet version - snippet.version += 1 - snippet.is_published = True - snippet.workflow_id = workflow.id - snippet.updated_by = account.id - session.add(snippet) - + WorkflowAgentPublishService.validate_publication_state(agents, skills=self._skills) + with self._publication.publication(source) as transaction: + agent_service = WorkflowAgentPublishService(repository=transaction.bindings) + agent_service.validate_prepared_publication(draft_workflow=transaction.draft, prepared=agents) + workflow = transaction.create_version(account_id=account.id) + agent_service.copy_agent_node_bindings_to_published( + draft_workflow=transaction.draft, published_workflow=workflow_binding_scope(workflow) + ) + transaction.activate(workflow, account_id=account.id) return workflow def get_all_published_workflows( @@ -765,7 +566,7 @@ def get_all_published_workflows( snippet: CustomizedSnippet, page: int, limit: int, - ) -> tuple[Sequence[Workflow], bool]: + ) -> tuple[Sequence[WorkflowRecord], bool]: """ Get all published workflow versions for snippet. @@ -775,27 +576,17 @@ def get_all_published_workflows( :param limit: Items per page :return: Tuple of (workflows list, has_more flag) """ - if not snippet.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where( - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version != "draft", - ) - .order_by(Workflow.version.desc()) - .limit(limit + 1) - .offset((page - 1) * limit) + workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( + session=session, + app_model=snippet, + page=page, + limit=limit, + user_id=None, + kind=WorkflowKind.SNIPPET, + include_draft=False, ) - workflows = list(session.scalars(stmt).all()) - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more + return [workflow_record(workflow, session) for workflow in workflows], has_more def update_workflow( self, @@ -805,7 +596,7 @@ def update_workflow( workflow_id: str, account: Account, data: dict[str, Any], - ) -> Workflow | None: + ) -> WorkflowRecord | None: """ Update a published snippet workflow version's display metadata. @@ -816,73 +607,15 @@ def update_workflow( :param data: Dictionary containing fields to update :return: Updated workflow or None if not found """ - stmt = select(Workflow).where( - Workflow.id == workflow_id, - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - Workflow.version != Workflow.VERSION_DRAFT, - ) - workflow = session.scalar(stmt) - if not workflow: - return None - - allowed_fields = {"marked_name", "marked_comment"} - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account.id - workflow.updated_at = datetime.now(UTC).replace(tzinfo=None) - session.add(workflow) - return workflow - - def delete_workflow( - self, - *, - session: Session, - snippet: CustomizedSnippet, - workflow_id: str, - ) -> bool: - """ - Delete a published snippet workflow version. - - :param session: Database session - :param snippet: CustomizedSnippet instance - :param workflow_id: Workflow ID - :return: True if successful - :raises: ValueError if workflow not found - :raises: WorkflowInUseError if workflow is the snippet's active version or published as a tool - :raises: DraftWorkflowDeletionError if workflow is a draft version - """ - stmt = select(Workflow).where( - Workflow.id == workflow_id, - Workflow.tenant_id == snippet.tenant_id, - Workflow.app_id == snippet.id, - self._snippet_kind_filter(), - ) - workflow = session.scalar(stmt) - if not workflow: - raise ValueError(f"Workflow with ID {workflow_id} not found") - - if workflow.version == Workflow.VERSION_DRAFT: - raise DraftWorkflowDeletionError("Cannot delete draft workflow versions") - - if snippet.workflow_id == workflow.id: - raise WorkflowInUseError(f"Cannot delete workflow that is currently in use by snippet '{snippet.id}'") - - tool_provider = session.scalar( - select(WorkflowToolProvider).where( - WorkflowToolProvider.tenant_id == snippet.tenant_id, - WorkflowToolProvider.app_id == snippet.id, - WorkflowToolProvider.version == workflow.version, - ) + workflow = WorkflowDefinitionStore.update_workflow( + session=session, + account_id=account.id, + data=data, + workflow_ref=WorkflowRef(tenant_id=snippet.tenant_id, owner_id=snippet.id, workflow_id=workflow_id), + kind=WorkflowKind.SNIPPET.value, + published_only=True, ) - if tool_provider: - raise WorkflowInUseError("Cannot delete workflow that is published as a tool") - - session.delete(workflow) - return True + return workflow_record(workflow, session) if workflow is not None else None # --- Default Block Configs --- @@ -971,10 +704,7 @@ def get_snippet_workflow_run( ) def get_snippet_workflow_run_node_executions( - self, - *, - snippet: CustomizedSnippet, - run_id: str, + self, *, snippet: CustomizedSnippet, run_id: str, tool_providers: ToolProviders ) -> list[WorkflowNodeExecutionTrace]: """ Get workflow run node execution list. @@ -995,7 +725,7 @@ def get_snippet_workflow_run_node_executions( with self._session_scope() as session: return assemble_workflow_node_execution_traces( - node_executions, self._node_execution_service_repo, session=session + node_executions, self._node_execution_service_repo, session=session, tool_providers=tool_providers ) # --- Node Execution Operations --- diff --git a/api/services/trial_app_generation_adapters.py b/api/services/trial_app_generation_adapters.py index c151f12a822ab8..bddb9ebead5890 100644 --- a/api/services/trial_app_generation_adapters.py +++ b/api/services/trial_app_generation_adapters.py @@ -13,17 +13,27 @@ from core.app.entities.app_invoke_entities import InvokeFrom from models import Account, App from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.trial_app_access_service import TrialAppRef from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables logger = logging.getLogger(__name__) class AppGenerateServiceRuntime(TrialAppGenerationRuntime): - def __init__(self, *, session_factory: sessionmaker[Session]) -> None: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: self._session_factory: sessionmaker[Session] = session_factory + self._variables = variables + self._runtime = runtime @override def generate( @@ -47,6 +57,8 @@ def generate( # receives its own session, never the admission or usage transaction. with self._session_factory(expire_on_commit=False) as session: generated_response: GenerationResponse = AppGenerateService.generate( + variables=self._variables, + runtime=self._runtime, app_model=app_model, user=account, args=args, diff --git a/api/services/trigger/errors.py b/api/services/trigger/errors.py new file mode 100644 index 00000000000000..6f56ebf08d28de --- /dev/null +++ b/api/services/trigger/errors.py @@ -0,0 +1,5 @@ +"""Framework-neutral trigger failures.""" + + +class WebhookBodyTooLargeError(Exception): + """The webhook payload exceeds the configured admission limit.""" diff --git a/api/services/trigger/schedule_service.py b/api/services/trigger/schedule_service.py index 29c01440378f3e..cf73900efbe1c4 100644 --- a/api/services/trigger/schedule_service.py +++ b/api/services/trigger/schedule_service.py @@ -1,123 +1,23 @@ -import json import logging -from datetime import datetime from sqlalchemy import select from sqlalchemy.orm import Session -from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE from core.workflow.nodes.trigger_schedule.entities import ( ScheduleConfig, - SchedulePlanUpdate, TriggerScheduleNodeData, VisualConfig, ) -from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError, ScheduleNotFoundError +from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError from graphon.entities.graph_config import NodeConfigDict -from libs.schedule_utils import calculate_next_run_at, convert_12h_to_24h from models.account import Account, TenantAccountJoin -from models.trigger import WorkflowSchedulePlan -from models.workflow import Workflow from services.account_errors import AccountNotFoundError +from services.trigger.schedule_policy import visual_to_cron logger = logging.getLogger(__name__) class ScheduleService: - @staticmethod - def create_schedule( - tenant_id: str, app_id: str, config: ScheduleConfig, *, session: Session - ) -> WorkflowSchedulePlan: - """ - Create a new schedule with validated configuration. - - Args: - session: Database session - tenant_id: Tenant ID - app_id: Application ID - config: Validated schedule configuration - - Returns: - Created WorkflowSchedulePlan instance - """ - next_run_at = calculate_next_run_at( - config.cron_expression, - config.timezone, - ) - - schedule = WorkflowSchedulePlan( - tenant_id=tenant_id, - app_id=app_id, - node_id=config.node_id, - cron_expression=config.cron_expression, - timezone=config.timezone, - next_run_at=next_run_at, - ) - - session.add(schedule) - session.flush() - - return schedule - - @staticmethod - def update_schedule(schedule_id: str, updates: SchedulePlanUpdate, *, session: Session) -> WorkflowSchedulePlan: - """ - Update an existing schedule with validated configuration. - - Args: - session: Database session - schedule_id: Schedule ID to update - updates: Validated update configuration - - Raises: - ScheduleNotFoundError: If schedule not found - - Returns: - Updated WorkflowSchedulePlan instance - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - # If time-related fields are updated, synchronously update the next_run_at. - time_fields_updated = False - - if updates.node_id is not None: - schedule.node_id = updates.node_id - - if updates.cron_expression is not None: - schedule.cron_expression = updates.cron_expression - time_fields_updated = True - - if updates.timezone is not None: - schedule.timezone = updates.timezone - time_fields_updated = True - - if time_fields_updated: - schedule.next_run_at = calculate_next_run_at( - schedule.cron_expression, - schedule.timezone, - ) - - session.flush() - return schedule - - @staticmethod - def delete_schedule(schedule_id: str, *, session: Session) -> None: - """ - Delete a schedule plan. - - Args: - session: Database session - schedule_id: Schedule ID to delete - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - session.delete(schedule) - session.flush() - @staticmethod def get_tenant_owner(tenant_id: str, *, session: Session) -> Account: """ @@ -146,26 +46,6 @@ def get_tenant_owner(tenant_id: str, *, session: Session) -> Account: else: raise AccountNotFoundError(f"Account not found for tenant: {tenant_id}") - @staticmethod - def update_next_run_at(schedule_id: str, *, session: Session) -> datetime: - """ - Advances the schedule to its next execution time after a successful trigger. - Uses current time as base to prevent missing executions during delays. - """ - schedule = session.get(WorkflowSchedulePlan, schedule_id) - if not schedule: - raise ScheduleNotFoundError(f"Schedule not found: {schedule_id}") - - # Base on current time to handle execution delays gracefully - next_run_at = calculate_next_run_at( - schedule.cron_expression, - schedule.timezone, - ) - - schedule.next_run_at = next_run_at - session.flush() - return next_run_at - @staticmethod def to_schedule_config(node_config: NodeConfigDict) -> ScheduleConfig: """ @@ -187,119 +67,9 @@ def to_schedule_config(node_config: NodeConfigDict) -> ScheduleConfig: if not frequency: raise ScheduleConfigError("Frequency is required for visual mode") visual_config = VisualConfig.model_validate(node_data.visual_config or {}) - cron_expression = ScheduleService.visual_to_cron(frequency=frequency, visual_config=visual_config) + cron_expression = visual_to_cron(frequency=frequency, visual_config=visual_config) if not cron_expression: raise ScheduleConfigError("Cron expression is required for visual mode") else: raise ScheduleConfigError(f"Invalid schedule mode: {mode}") return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone) - - @staticmethod - def extract_schedule_config(workflow: Workflow) -> ScheduleConfig | None: - """ - Extracts schedule configuration from workflow graph. - - Searches for the first schedule trigger node in the workflow and converts - its configuration (either visual or cron mode) into a unified ScheduleConfig. - - Args: - workflow: The workflow containing the graph definition - - Returns: - ScheduleConfig if a valid schedule node is found, None if no schedule node exists - - Raises: - ScheduleConfigError: If graph parsing fails or schedule configuration is invalid - - Note: - Currently only returns the first schedule node found. - Multiple schedule nodes in the same workflow are not supported. - """ - try: - graph_data = workflow.graph_dict - except (json.JSONDecodeError, TypeError, AttributeError) as e: - raise ScheduleConfigError(f"Failed to parse workflow graph: {e}") from e - - if not graph_data: - raise ScheduleConfigError("Workflow graph is empty") - - nodes = graph_data.get("nodes", []) - for node in nodes: - node_data = node.get("data", {}) - - if node_data.get("type") != TRIGGER_SCHEDULE_NODE_TYPE: - continue - - node_id = node.get("id", "start") - trigger_data = TriggerScheduleNodeData.model_validate(node_data) - mode = trigger_data.mode - timezone = trigger_data.timezone - - cron_expression = None - if mode == "cron": - cron_expression = trigger_data.cron_expression - if not cron_expression: - raise ScheduleConfigError("Cron expression is required for cron mode") - elif mode == "visual": - frequency = trigger_data.frequency - if not frequency: - raise ScheduleConfigError("Frequency is required for visual mode") - visual_config = VisualConfig.model_validate(trigger_data.visual_config or {}) - cron_expression = ScheduleService.visual_to_cron(frequency, visual_config) - else: - raise ScheduleConfigError(f"Invalid schedule mode: {mode}") - - return ScheduleConfig(node_id=node_id, cron_expression=cron_expression, timezone=timezone) - - return None - - @staticmethod - def visual_to_cron(frequency: str, visual_config: VisualConfig) -> str: - """ - Converts user-friendly visual schedule settings to cron expression. - Maintains consistency with frontend UI expectations while supporting croniter's extended syntax. - """ - if frequency == "hourly": - if visual_config.on_minute is None: - raise ScheduleConfigError("on_minute is required for hourly schedules") - return f"{visual_config.on_minute} * * * *" - - elif frequency == "daily": - if not visual_config.time: - raise ScheduleConfigError("time is required for daily schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - return f"{minute} {hour} * * *" - - elif frequency == "weekly": - if not visual_config.time: - raise ScheduleConfigError("time is required for weekly schedules") - if not visual_config.weekdays: - raise ScheduleConfigError("Weekdays are required for weekly schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - weekday_map = {"sun": "0", "mon": "1", "tue": "2", "wed": "3", "thu": "4", "fri": "5", "sat": "6"} - cron_weekdays = [weekday_map[day] for day in visual_config.weekdays] - return f"{minute} {hour} * * {','.join(sorted(cron_weekdays))}" - - elif frequency == "monthly": - if not visual_config.time: - raise ScheduleConfigError("time is required for monthly schedules") - if not visual_config.monthly_days: - raise ScheduleConfigError("Monthly days are required for monthly schedules") - hour, minute = convert_12h_to_24h(visual_config.time) - - numeric_days: list[int] = [] - has_last = False - for day in visual_config.monthly_days: - if day == "last": - has_last = True - else: - numeric_days.append(day) - - result_days = [str(d) for d in sorted(set(numeric_days))] - if has_last: - result_days.append("L") - - return f"{minute} {hour} {','.join(result_days)} * *" - - else: - raise ScheduleConfigError(f"Unsupported frequency: {frequency}") diff --git a/api/services/trigger/trigger_service.py b/api/services/trigger/trigger_service.py index 911331e357c28b..cec7945505f2a5 100644 --- a/api/services/trigger/trigger_service.py +++ b/api/services/trigger/trigger_service.py @@ -1,30 +1,20 @@ import logging import secrets import time -from collections.abc import Mapping -from typing import Any from flask import Request, Response -from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.orm import sessionmaker from core.plugin.entities.plugin_daemon import CredentialType from core.plugin.entities.request import TriggerDispatchResponse, TriggerInvokeEventResponse from core.plugin.impl.exc import PluginNotFoundError -from core.trigger.constants import TRIGGER_PLUGIN_NODE_TYPE from core.trigger.debug.events import PluginTriggerDebugEvent from core.trigger.provider import PluginTriggerProviderController from core.trigger.trigger_manager import TriggerManager from core.trigger.utils.encryption import create_trigger_provider_encrypter_for_subscription from core.workflow.nodes.trigger_plugin.entities import TriggerEventNodeData -from extensions.ext_database import db -from extensions.ext_redis import redis_client from graphon.entities.graph_config import NodeConfigDict -from models.model import App from models.provider_ids import TriggerProviderID -from models.trigger import TriggerSubscription, WorkflowPluginTrigger -from models.workflow import Workflow +from models.trigger import TriggerSubscription from services.trigger.trigger_provider_service import TriggerProviderService from services.trigger.trigger_request_service import TriggerHttpRequestCachingService from services.workflow.entities import PluginTriggerDispatchData @@ -37,8 +27,6 @@ class TriggerService: __TEMPORARY_ENDPOINT_EXPIRE_MS__ = 5 * 60 * 1000 __ENDPOINT_REQUEST_CACHE_COUNT__ = 10 __ENDPOINT_REQUEST_CACHE_EXPIRE_MS__ = 5 * 60 * 1000 - __PLUGIN_TRIGGER_NODE_CACHE_KEY__ = "plugin_trigger_nodes" - MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW = 5 # Maximum allowed plugin trigger nodes per workflow @classmethod def invoke_trigger_event( @@ -148,155 +136,3 @@ def process_endpoint(cls, endpoint_id: str, request: Request) -> Response | None request_id, ) return dispatch_response.response - - @classmethod - def sync_plugin_trigger_relationships(cls, app: App, workflow: Workflow): - """ - Sync plugin trigger relationships in DB. - - 1. Check if the workflow has any plugin trigger nodes - 2. Fetch the nodes from DB, see if there were any plugin trigger records already - 3. Diff the nodes and the plugin trigger records, create/update/delete the records as needed - - Approach: - Frequent DB operations may cause performance issues, using Redis to cache it instead. - If any record exists, cache it. - - Limits: - - Maximum 5 plugin trigger nodes per workflow - """ - - class Cache(BaseModel): - """ - Cache model for plugin trigger nodes - """ - - record_id: str - node_id: str - provider_id: str - event_name: str - subscription_id: str - - # Walk nodes to find plugin triggers - nodes_in_graph: list[Mapping[str, Any]] = [] - for node_id, node_config in workflow.walk_nodes(TRIGGER_PLUGIN_NODE_TYPE): - # Extract plugin trigger configuration from node - plugin_id = node_config.get("plugin_id", "") - provider_id = node_config.get("provider_id", "") - event_name = node_config.get("event_name", "") - subscription_id = node_config.get("subscription_id", "") - - if not subscription_id: - continue - - nodes_in_graph.append( - { - "node_id": node_id, - "plugin_id": plugin_id, - "provider_id": provider_id, - "event_name": event_name, - "subscription_id": subscription_id, - } - ) - - # Check plugin trigger node limit - if len(nodes_in_graph) > cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW: - raise ValueError( - f"Workflow exceeds maximum plugin trigger node limit. " - f"Found {len(nodes_in_graph)} plugin trigger nodes, " - f"maximum allowed is {cls.MAX_PLUGIN_TRIGGER_NODES_PER_WORKFLOW}" - ) - - not_found_in_cache: list[Mapping[str, Any]] = [] - for node_info in nodes_in_graph: - node_id = node_info["node_id"] - # firstly check if the node exists in cache - if not redis_client.get(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}"): - not_found_in_cache.append(node_info) - continue - - with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session: - try: - # lock the concurrent plugin trigger creation - redis_client.lock(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock", timeout=10) - # fetch the non-cached nodes from DB - all_records = session.scalars( - select(WorkflowPluginTrigger).where( - WorkflowPluginTrigger.app_id == app.id, - WorkflowPluginTrigger.tenant_id == app.tenant_id, - ) - ).all() - - nodes_id_in_db = {node.node_id: node for node in all_records} - nodes_id_in_graph = {node["node_id"] for node in nodes_in_graph} - - # get the nodes not found both in cache and DB - nodes_not_found = [ - node_info for node_info in not_found_in_cache if node_info["node_id"] not in nodes_id_in_db - ] - - # create new plugin trigger records - for node_info in nodes_not_found: - plugin_trigger = WorkflowPluginTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id=node_info["node_id"], - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - session.add(plugin_trigger) - session.flush() # Get the ID for caching - - cache = Cache( - record_id=plugin_trigger.id, - node_id=node_info["node_id"], - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - redis_client.set( - f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_info['node_id']}", - cache.model_dump_json(), - ex=60 * 60, - ) - - # Update existing records if subscription_id changed - for node_info in nodes_in_graph: - node_id = node_info["node_id"] - if node_id in nodes_id_in_db: - existing_record = nodes_id_in_db[node_id] - if ( - existing_record.subscription_id != node_info["subscription_id"] - or existing_record.provider_id != node_info["provider_id"] - or existing_record.event_name != node_info["event_name"] - ): - existing_record.subscription_id = node_info["subscription_id"] - existing_record.provider_id = node_info["provider_id"] - existing_record.event_name = node_info["event_name"] - session.add(existing_record) - - # Update cache - cache = Cache( - record_id=existing_record.id, - node_id=node_id, - provider_id=node_info["provider_id"], - event_name=node_info["event_name"], - subscription_id=node_info["subscription_id"], - ) - redis_client.set( - f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}", - cache.model_dump_json(), - ex=60 * 60, - ) - - # delete the nodes not found in the graph - for node_id in nodes_id_in_db: - if node_id not in nodes_id_in_graph: - session.delete(nodes_id_in_db[node_id]) - redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:{app.id}:{node_id}") - except Exception: - logger.exception("Failed to sync plugin trigger relationships for app %s", app.id) - raise - finally: - redis_client.delete(f"{cls.__PLUGIN_TRIGGER_NODE_CACHE_KEY__}:apps:{app.id}:lock") diff --git a/api/services/trigger/webhook_service.py b/api/services/trigger/webhook_service.py index 0c9f6f8473ffac..d7fc3f345a4433 100644 --- a/api/services/trigger/webhook_service.py +++ b/api/services/trigger/webhook_service.py @@ -2,22 +2,18 @@ import logging import math import mimetypes -import secrets from collections.abc import Callable, Mapping, Sequence from typing import Any, NotRequired, Protocol, TypedDict import orjson from flask import request -from pydantic import BaseModel from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import RequestEntityTooLarge from configs import dify_config from core.app.file_access import DatabaseFileAccessController from core.tools.tool_file_manager import ToolFileManager -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from core.workflow.nodes.trigger_webhook.entities import ( ContentType, WebhookBodyParameter, @@ -26,7 +22,6 @@ ) from enums import QuotaType from extensions.ext_database import db -from extensions.ext_redis import redis_client from factories import file_factory from graphon.entities.graph_config import NodeConfigDict from graphon.file import FileTransferMethod @@ -35,10 +30,13 @@ from models.model import App, EndUser from models.trigger import AppTrigger, WorkflowWebhookTrigger from models.workflow import Workflow +from repositories.trigger.workflow_repository import WorkflowTriggerRepository from services.async_workflow_service import AsyncWorkflowService from services.errors.app import QuotaExceededError from services.quota_service import QuotaService from services.trigger.app_trigger_service import AppTriggerService +from services.trigger.errors import WebhookBodyTooLargeError +from services.trigger.workflow_policy import webhook_node_ids from services.workflow.entities import WebhookTriggerData from services.workflow_service import WorkflowService @@ -84,9 +82,6 @@ def get_or_create_end_user_by_type( class WebhookService: """Service for handling webhook operations.""" - __WEBHOOK_NODE_CACHE_KEY__ = "webhook_nodes" - MAX_WEBHOOK_NODES_PER_WORKFLOW = 5 # Maximum allowed webhook nodes per workflow - @staticmethod def _sanitize_key(key: str) -> str: """Normalize external keys (headers/params) to workflow-safe variables.""" @@ -292,7 +287,7 @@ def _validate_content_length(cls) -> None: """Validate request content length against maximum allowed size.""" content_length = request.content_length if content_length and content_length > dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE: - raise RequestEntityTooLarge( + raise WebhookBodyTooLargeError( f"Webhook request too large: {content_length} bytes exceeds maximum allowed size " f"of {dify_config.WEBHOOK_REQUEST_BODY_MAX_SIZE} bytes" ) @@ -906,117 +901,24 @@ def generate_webhook_response(cls, node_config: NodeConfigDict) -> tuple[dict[st return response_data, status_code @classmethod - def sync_webhook_relationships(cls, app: App, workflow: Workflow, *, remove_stale: bool = True): - """ - Sync webhook relationships in DB. - - 1. Check if the workflow has any webhook trigger nodes - 2. Fetch the nodes from DB, see if there were any webhook records already - 3. Diff the nodes and the webhook records, creating missing records and optionally deleting stale records - - Draft workflow synchronization preserves stale records so undo can restore a - webhook node without changing its URL. Published workflow synchronization - removes stale records after the deletion becomes effective. - - Approach: - Frequent DB operations may cause performance issues, using Redis to cache it instead. - If any record exists, cache it. - - Limits: - - Maximum 5 webhook nodes per workflow - """ - - class Cache(BaseModel): - """ - Cache model for webhook nodes - """ - - record_id: str - node_id: str - webhook_id: str - - nodes_id_in_graph = [node_id for node_id, _ in workflow.walk_nodes(TRIGGER_WEBHOOK_NODE_TYPE)] - - # Check webhook node limit - if len(nodes_id_in_graph) > cls.MAX_WEBHOOK_NODES_PER_WORKFLOW: - raise ValueError( - f"Workflow exceeds maximum webhook node limit. " - f"Found {len(nodes_id_in_graph)} webhook nodes, maximum allowed is {cls.MAX_WEBHOOK_NODES_PER_WORKFLOW}" + def sync_webhook_relationships( + cls, + app: App, + workflow: Workflow, + *, + remove_stale: bool = True, + session_factory: sessionmaker[Session] | None = None, + ) -> None: + """Allocate draft URLs with the same row lock and persistence used by publication.""" + if app.created_by is None: + raise RuntimeError("App is missing created_by; cannot create webhook trigger") + node_ids = webhook_node_ids(workflow.graph_dict.get("nodes", [])) + with (session_factory or sessionmaker(db.engine)).begin() as session: + WorkflowTriggerRepository.lock_app(session, app) + WorkflowTriggerRepository.sync_webhooks( + session, + app, + node_ids, + created_by=app.created_by, + remove_stale=remove_stale, ) - - not_found_in_cache: list[str] = [] - for node_id in nodes_id_in_graph: - # firstly check if the node exists in cache - if not redis_client.get(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}"): - not_found_in_cache.append(node_id) - continue - - lock_key = f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:apps:{app.id}:lock" - lock = redis_client.lock(lock_key, timeout=10) - lock_acquired = False - - try: - # acquire the lock with blocking and timeout - lock_acquired = lock.acquire(blocking=True, blocking_timeout=10) - if not lock_acquired: - logger.warning("Failed to acquire lock for webhook sync, app %s", app.id) - raise RuntimeError("Failed to acquire lock for webhook trigger synchronization") - - with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session: - # fetch the non-cached nodes from DB - all_records = session.scalars( - select(WorkflowWebhookTrigger).where( - WorkflowWebhookTrigger.app_id == app.id, - WorkflowWebhookTrigger.tenant_id == app.tenant_id, - ) - ).all() - - nodes_id_in_db = {node.node_id: node for node in all_records} - - # get the nodes not found both in cache and DB - nodes_not_found = [node_id for node_id in not_found_in_cache if node_id not in nodes_id_in_db] - - # create new webhook records - for node_id in nodes_not_found: - if app.created_by is None: - raise RuntimeError("App is missing created_by; cannot create webhook trigger") - webhook_record = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id=node_id, - webhook_id=cls.generate_webhook_id(), - created_by=app.created_by, - ) - session.add(webhook_record) - session.flush() - cache = Cache(record_id=webhook_record.id, node_id=node_id, webhook_id=webhook_record.webhook_id) - redis_client.set( - f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}", cache.model_dump_json(), ex=60 * 60 - ) - - if remove_stale: - # Delete relationships only when reconciling an effective published workflow. - for node_id in nodes_id_in_db: - if node_id not in nodes_id_in_graph: - session.delete(nodes_id_in_db[node_id]) - redis_client.delete(f"{cls.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:{node_id}") - except Exception: - logger.exception("Failed to sync webhook relationships for app %s", app.id) - raise - finally: - # release the lock only if it was acquired - if lock_acquired: - try: - lock.release() - except Exception: - logger.exception("Failed to release lock for webhook sync, app %s", app.id) - - @classmethod - def generate_webhook_id(cls) -> str: - """ - Generate unique 24-character webhook ID - - Deduplication is not needed, DB already has unique constraint on webhook_id. - """ - # Generate 24-character random string - return secrets.token_urlsafe(18)[:24] # token_urlsafe gives base64url, take first 24 chars diff --git a/api/services/workflow/console_service.py b/api/services/workflow/console_service.py index bc7b736cbd63d2..22e74e33a21358 100644 --- a/api/services/workflow/console_service.py +++ b/api/services/workflow/console_service.py @@ -94,7 +94,7 @@ def last_run(self, context: RequestContext, app_id: str, node_id: str) -> dict[s def poll_trigger( self, context: RequestContext, app_id: str, node_ids: list[str], *, single_node: bool, select_all: bool ) -> WorkflowTriggerEvent | None: ... - def stop(self, task_id: str) -> None: ... + def stop(self, context: RequestContext, app_id: str, task_id: str) -> None: ... def default_blocks(self) -> Sequence[Mapping[str, object]]: ... def default_block(self, block_type: str, filters: dict[str, Any] | None) -> Mapping[str, object] | None: ... def retire_agents(self, context: RequestContext, agent_ids: list[str]) -> None: ... @@ -253,8 +253,8 @@ def trigger( "An unexpected error occurred while running the node." if single_node else None ) from None - def stop(self, task_id: str) -> None: - self._runtime.stop(task_id) + def stop(self, context: RequestContext, app_id: str, task_id: str) -> None: + self._runtime.stop(context, app_id, task_id) def default_blocks(self) -> Sequence[Mapping[str, object]]: return self._runtime.default_blocks() diff --git a/api/services/workflow/contracts.py b/api/services/workflow/contracts.py index cdf1a9fb28ae58..52617aa3d197a0 100644 --- a/api/services/workflow/contracts.py +++ b/api/services/workflow/contracts.py @@ -325,6 +325,7 @@ def sync_plugins(self, relationships: Sequence[Mapping[str, Any]]) -> None: ... def sync_schedule(self, schedule: ScheduleConfig | None) -> None: ... def sync_triggers(self, triggers: Sequence[Mapping[str, Any]]) -> None: ... def sync_datasets(self, dataset_ids: set[str]) -> None: ... + def sync_knowledge_spaces(self, workflow: WorkflowSnapshot) -> None: ... def activate(self, workflow: WorkflowSnapshot) -> None: ... diff --git a/api/services/workflow/definition_gateway.py b/api/services/workflow/definition_gateway.py index 49c273536c44c6..b150a33b70c928 100644 --- a/api/services/workflow/definition_gateway.py +++ b/api/services/workflow/definition_gateway.py @@ -43,7 +43,7 @@ def snapshot( class WorkflowDefinitionValidator(Protocol): - def validate_features_structure(self, app_model: App, features: dict[str, Any]) -> None: ... + def validate_features_structure(self, app_model: App, features: dict[str, Any]) -> dict[str, Any | None]: ... def validate_graph_structure(self, graph: Mapping[str, Any]) -> None: ... def validate_publication( self, app_model: App, draft_workflow: Workflow, *, credentials: CredentialQuery diff --git a/api/services/workflow/dsl_import.py b/api/services/workflow/dsl_import.py index 50019231fdc3ed..ed801b7f9f5dc6 100644 --- a/api/services/workflow/dsl_import.py +++ b/api/services/workflow/dsl_import.py @@ -25,6 +25,7 @@ from services.dsl_content import DSL_MAX_SIZE, dsl_content_size from services.entities.dsl_entities import ImportMode from services.errors.app import DraftChangedError +from services.workflow.contracts import WorkflowRecord from services.workflow.graph_check import ( GRAPH_MODES, GraphIssue, @@ -49,7 +50,7 @@ def __init__(self, issues: Sequence[GraphIssue]) -> None: self.issues = list(issues) -def stored_secret_ids(draft: Workflow) -> set[str]: +def stored_secret_ids(draft: Workflow | WorkflowRecord) -> set[str]: return {variable.id for variable in draft.environment_variables if isinstance(variable, SecretVariable)} diff --git a/api/services/workflow/execution/adapters/agent_message_transformer.py b/api/services/workflow/execution/adapters/agent_message_transformer.py index a8d85a53abfadd..25373c2d65cbf7 100644 --- a/api/services/workflow/execution/adapters/agent_message_transformer.py +++ b/api/services/workflow/execution/adapters/agent_message_transformer.py @@ -52,7 +52,6 @@ def transform( node_id: str, node_execution_id: str, ) -> Generator[NodeEventPayload]: - message_stream = ToolFileMessageTransformer.transform_tool_invoke_messages( messages=messages, user_id=user_id, diff --git a/api/services/workflow/execution/adapters/agent_node.py b/api/services/workflow/execution/adapters/agent_node.py index 1befe29af0cb22..8fadf8f8461848 100644 --- a/api/services/workflow/execution/adapters/agent_node.py +++ b/api/services/workflow/execution/adapters/agent_node.py @@ -11,7 +11,6 @@ AgentInvocationError, AgentMessageTransformError, ) -from core.workflow.nodes.agent.message_transformer import AgentMessageTransformer from core.workflow.nodes.agent.strategy_protocols import AgentStrategyPresentationProvider, AgentStrategyResolver from core.workflow.system_variables import SystemVariableKey, get_system_text from graphon.engine_events import NodeEvent @@ -19,6 +18,7 @@ from graphon.node_events import NodeEventPayload, NodeRunResult, StreamCompletedEvent from graphon.nodes.base.node import Node from graphon.nodes.base.variable_template_parser import VariableTemplateParser +from services.workflow.execution.adapters.agent_message_transformer import AgentMessageTransformer from services.workflow.execution.adapters.agent_runtime import AgentRuntimeSupport if TYPE_CHECKING: diff --git a/api/services/workflow/execution/adapters/chatflow/generate_task_pipeline.py b/api/services/workflow/execution/adapters/chatflow/generate_task_pipeline.py index 1e601522d7e383..0ae32e7d266061 100644 --- a/api/services/workflow/execution/adapters/chatflow/generate_task_pipeline.py +++ b/api/services/workflow/execution/adapters/chatflow/generate_task_pipeline.py @@ -281,7 +281,7 @@ def _build_paused_blocking_response_from_human_input( paused_nodes=paused_nodes, reasons=reasons, status=WorkflowExecutionStatus.PAUSED, - elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at, + elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(), total_tokens=runtime_state.total_tokens, total_steps=runtime_state.node_run_steps, ), diff --git a/api/services/workflow/execution/adapters/workflow/generate_task_pipeline.py b/api/services/workflow/execution/adapters/workflow/generate_task_pipeline.py index 2bec0d4f1a5cad..5968015f24fdb9 100644 --- a/api/services/workflow/execution/adapters/workflow/generate_task_pipeline.py +++ b/api/services/workflow/execution/adapters/workflow/generate_task_pipeline.py @@ -1,5 +1,4 @@ import logging -import time from collections.abc import Callable, Generator from core.app.apps.base_app_queue_manager import AppQueueManager @@ -219,7 +218,7 @@ def _build_paused_blocking_response_from_human_input( status=WorkflowExecutionStatus.PAUSED, outputs={}, error=None, - elapsed_time=time.perf_counter() - self._base_task_pipeline.start_at, + elapsed_time=self._application_generate_entity.workflow_execution_timing.finish(), total_tokens=runtime_state.total_tokens, total_steps=runtime_state.node_run_steps, created_at=created_at, diff --git a/api/services/workflow/execution/adapters/workflow_entry.py b/api/services/workflow/execution/adapters/workflow_entry.py index 7958fdc877286a..80b548465d785d 100644 --- a/api/services/workflow/execution/adapters/workflow_entry.py +++ b/api/services/workflow/execution/adapters/workflow_entry.py @@ -50,7 +50,6 @@ from graphon.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader, load_into_variable_pool from models import Account from models.agent_runtime_contracts import WorkflowAgentRuntimeBindings -from models.workflow import Workflow from services.human_input.ports import HumanInputFormReader from services.workflow.contracts import WorkflowSnapshot from services.workflow.execution.adapters.human_input_events import HumanInputFormEventFilter @@ -230,7 +229,7 @@ def run(self) -> Generator[EngineEvent]: def single_step_run( cls, *, - workflow: WorkflowSnapshot | Workflow, + workflow: WorkflowSnapshot, node_id: str, user_id: str, user_inputs: Mapping[str, Any], diff --git a/api/services/workflow/graph_check.py b/api/services/workflow/graph_check.py index 03088b51e7e018..9365971108df39 100644 --- a/api/services/workflow/graph_check.py +++ b/api/services/workflow/graph_check.py @@ -9,7 +9,6 @@ from typing import Any, Final from pydantic import ValidationError -from sqlalchemy.orm import Session from core.trigger.constants import TRIGGER_NODE_TYPES from core.workflow.human_input_adapter import adapt_node_config_for_graph @@ -20,6 +19,7 @@ from graphon.nodes.base.node import Node from graphon.variables import VariableBase from models import AppMode +from services.credentials.query import CredentialQuery from services.workflow.node_defaults import complete_sub_models, editor_defaults, node_data_type from services.workflow_service import WorkflowService from services.workflow_variable_reference_validator import SOURCE_HANDLE, branch_handles @@ -75,14 +75,16 @@ def refused_node_types(mode: AppMode) -> frozenset[str]: return frozenset({BuiltinNodeTypes.ANSWER}) -def credential_check(workspace_id: str, environment: Mapping[str, VariableBase], session: Session) -> ResourceCheck: +def credential_check( + workspace_id: str, environment: Mapping[str, VariableBase], credentials: CredentialQuery +) -> ResourceCheck: """A `ResourceCheck` reporting what publish's credential check would refuse for one node.""" service = WorkflowService() def check(node: Mapping[str, Any]) -> str | None: try: - service.validate_node_credentials(workspace_id, node, environment, session=session) + service.validate_node_credentials(workspace_id, node, environment, credentials=credentials) except ValueError as error: return str(error) return None diff --git a/api/services/workflow/graph_diff.py b/api/services/workflow/graph_diff.py index c26e7e61d5443c..252b84662bc96e 100644 --- a/api/services/workflow/graph_diff.py +++ b/api/services/workflow/graph_diff.py @@ -14,6 +14,7 @@ from libs import helper from models.model import App from models.workflow import Workflow +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.workflow.node_defaults import fill_node_data _DATA_UI_KEYS: Final = frozenset({"selected", "_connectedSourceHandleIds", "_connectedTargetHandleIds"}) @@ -98,13 +99,11 @@ def draft_token(draft: Workflow) -> str: def current_draft_token(tenant_id: str, app_id: str) -> str | None: """The app draft's `draft_token`, read in its own session; None when the app has no draft.""" - from services.workflow_service import WorkflowService - with session_factory.create_session() as session: app = session.get(App, app_id) if app is None or app.tenant_id != tenant_id: return None - draft = WorkflowService.get_draft_workflow(app, session=session) + draft = WorkflowDefinitionStore.get_draft_workflow(app, session=session) return draft_token(draft) if draft else None diff --git a/api/services/workflow/node_output_inspector_service.py b/api/services/workflow/node_output_inspector_service.py index 93d6ded71ef6cc..4bc50074f735a3 100644 --- a/api/services/workflow/node_output_inspector_service.py +++ b/api/services/workflow/node_output_inspector_service.py @@ -26,7 +26,7 @@ Cross-tenant / cross-app rows still 404 via the standard tenant/app scope. 3. **Declared outputs by node kind**: * Agent v2 nodes resolve their declared list via - :class:`WorkflowAgentBindingResolver` (the binding owns custom declarations; + :class:`WorkflowAgentRuntimeBindings` (the binding owns custom declarations; the system ``text`` output is derived for display). * Other node kinds don't have a declared-output schema yet; we surface the keys present in the execution payload as a best-effort list typed @@ -55,16 +55,13 @@ from sqlalchemy.orm import Session from core.app.file_access import DatabaseFileAccessController -from core.workflow.nodes.agent_v2.binding_resolver import ( - WorkflowAgentBindingError, - WorkflowAgentBindingResolver, -) from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data from factories.file_factory.builders import build_from_mapping from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus from graphon.file import helpers as file_helpers from models import App from models.agent_config_entities import DeclaredOutputConfig, DeclaredOutputType, effective_declared_outputs +from models.agent_runtime_contracts import WorkflowAgentBindingError, WorkflowAgentRuntimeBindings from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun logger = logging.getLogger(__name__) @@ -396,7 +393,7 @@ class NodeOutputInspectorService: """Read-only Inspector for draft + published workflow runs. The service is dependency-light: it holds a single - :class:`WorkflowAgentBindingResolver` so agent v2 nodes can map to their + :class:`WorkflowAgentRuntimeBindings` so agent v2 nodes can map to their declared outputs without re-implementing binding lookup. All other I/O receives an explicit SQLAlchemy session from its caller so transaction ownership stays at the controller/task boundary. @@ -405,8 +402,8 @@ class NodeOutputInspectorService: every load — the same scope guard regardless of trigger source. """ - def __init__(self, binding_resolver: WorkflowAgentBindingResolver | None = None) -> None: - self._binding_resolver = binding_resolver or WorkflowAgentBindingResolver() + def __init__(self, binding_resolver: WorkflowAgentRuntimeBindings) -> None: + self._binding_resolver = binding_resolver # ── public API ──────────────────────────────────────────────────────── diff --git a/api/services/workflow/publication_service.py b/api/services/workflow/publication_service.py index c396765faf079a..b3925ccc7d2dc5 100644 --- a/api/services/workflow/publication_service.py +++ b/api/services/workflow/publication_service.py @@ -61,6 +61,7 @@ def publish_workflow[BindingStore]( agent_service.validate_prepared_publication(draft_workflow=transaction.draft, prepared=agents) workflow = transaction.create_version(marked_name=marked_name, marked_comment=marked_comment) agent_service.copy_agent_node_bindings_to_published(draft_workflow=transaction.draft, published_workflow=workflow) + transaction.sync_knowledge_spaces(workflow) if configuration.has_trigger_runtime: transaction.sync_webhooks(configuration.webhook_nodes) transaction.sync_plugins(configuration.plugins) diff --git a/api/services/workflow/runtime_gateway.py b/api/services/workflow/runtime_gateway.py new file mode 100644 index 00000000000000..3b4c2ff5a3715d --- /dev/null +++ b/api/services/workflow/runtime_gateway.py @@ -0,0 +1,280 @@ +"""Adapt the existing workflow engine to framework-neutral Console ports.""" + +from collections.abc import Mapping, Sequence +from typing import Any, Protocol + +from sqlalchemy.orm import Session, sessionmaker + +from core.app.app_config.features.file_upload.manager import FileUploadConfigManager +from core.app.entities.app_invoke_entities import InvokeFrom +from core.app.file_access import DatabaseFileAccessController +from core.plugin.impl.exc import PluginInvokeError +from core.trigger.constants import TRIGGER_SCHEDULE_NODE_TYPE +from core.trigger.debug.event_selectors import create_event_poller, select_trigger_debug_events +from factories import file_factory +from graphon.file import File +from machinery.context import RequestContext +from models import Account, App +from models.enums import CreatorUserRole +from models.model import AppMode +from models.workflow import WorkflowNodeExecutionModel +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository +from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository +from services.agent.retirement_service import WorkflowAgentRetirementService +from services.app.generation.runtime import AppGenerationRuntime +from services.app_generate_service import AppGenerateService +from services.errors.app import WorkflowNotFoundError +from services.workflow.contracts import ( + WorkflowGeneration, + WorkflowSnapshot, + WorkflowTriggerError, + WorkflowTriggerEvent, +) +from services.workflow.node_execution_extras import node_execution_extras +from services.workflow.stream_reservation import WorkflowStreamReservation +from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow_service import WorkflowService + + +class WorkflowTaskControl(Protocol): + def stop_workflow_task( + self, + *, + tenant_id: str, + app_id: str, + task_id: str, + app_mode: AppMode, + owner: tuple[CreatorUserRole, str] | None, + ) -> bool: ... + + +class WorkflowRuntimeGateway: + def __init__( + self, + *, + session_factory: sessionmaker[Session], + workflows: WorkflowService, + definitions: WorkflowDefinitionRepository, + reservations: WorkflowDebugReservationRepository, + executions: WorkflowNodeExecutionRepository, + generator: type[AppGenerateService], + tasks: WorkflowTaskControl, + file_access: DatabaseFileAccessController, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, + ) -> None: + self._sessions = session_factory + self._workflows = workflows + self._definitions = definitions + self._reservations = reservations + self._executions = executions + self._generator = generator + self._tasks = tasks + self._file_access = file_access + self._variables = variables + self._runtime = runtime + + def generate( + self, + context: RequestContext, + app_id: str, + args: dict[str, Any], + *, + root_node_id: str | None, + workflow: WorkflowSnapshot | None, + ) -> WorkflowGeneration: + reservation = ( + WorkflowStreamReservation(lambda: self._cancel_trigger_debug(workflow, context.account_id)) + if workflow is not None and workflow.execution_id is not None + else None + ) + try: + app, actor, definition = self._definitions.debug_context( + context, app_id, workflow_id=args.get("workflow_id"), snapshot=workflow + ) + stream = self._generator.generate_workflow_stream( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=definition, + args=args, + invoke_from=InvokeFrom.DEBUGGER, + root_node_id=root_node_id, + workflow_snapshot=workflow, + reservation=reservation, + ) + return reservation.wrap(stream) if reservation is not None else stream + except Exception as error: + if reservation is not None: + reservation.close() + if isinstance(error, PluginInvokeError) and root_node_id is not None: + raise WorkflowTriggerError(error.to_user_friendly_error()) from error + raise + + def _cancel_trigger_debug(self, snapshot: WorkflowSnapshot, account_id: str) -> None: + if snapshot.execution_id is None: + return + WorkflowAgentRetirementService.finish_execution( + sessions=self._sessions, + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=account_id, + cancel=True, + ) + + def iteration( + self, context: RequestContext, app_id: str, node_id: str, inputs: dict[str, Any] | None + ) -> WorkflowGeneration: + app, actor, workflow = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + return self._generator.generate_single_iteration( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=workflow, + node_id=node_id, + args={"inputs": inputs} if inputs is not None else {}, + streaming=True, + ) + + def loop( + self, context: RequestContext, app_id: str, node_id: str, inputs: dict[str, Any] | None + ) -> WorkflowGeneration: + app, actor, workflow = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + return self._generator.generate_single_loop( + runtime=self._runtime, + variables=self._variables, + app_model=app, + user=actor, + workflow=workflow, + node_id=node_id, + args={"inputs": inputs}, + streaming=True, + ) + + def _draft( + self, context: RequestContext, app_id: str, missing_message: str + ) -> tuple[App, Account, WorkflowSnapshot]: + try: + return self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=None) + except ValueError as error: + raise ValueError(missing_message) from error + + def _files(self, workflow: WorkflowSnapshot, files: list[dict[str, Any]] | None) -> Sequence[File]: + config = FileUploadConfigManager.convert(workflow.features_dict, is_vision=False) + if config is None: + return [] + return file_factory.build_from_mappings( + mappings=files or [], + tenant_id=workflow.tenant_id, + config=config, + access_controller=self._file_access, + sessions=self._sessions, + ) + + def _execution(self, execution: WorkflowNodeExecutionModel, *, include_details: bool) -> dict[str, Any]: + detail = self._executions.execution_record(execution, include_details=include_details) + if include_details: + # Plugin icons can perform external I/O; repository reads are closed. + detail["extras"] = node_execution_extras(execution, tool_providers=self._runtime.tool_providers) + return detail + + def run_node( + self, + context: RequestContext, + app_id: str, + node_id: str, + args: dict[str, Any], + *, + include_details: bool, + workflow: WorkflowSnapshot | None, + ) -> dict[str, Any]: + if workflow is None: + app, actor, draft = self._draft(context, app_id, "Workflow not initialized") + else: + app, actor, draft = self._definitions.debug_context(context, app_id, workflow_id=None, snapshot=workflow) + raw_files = args.get("files") + execution = self._workflows.run_draft_workflow_node( + variables=self._variables, + app_model=app, + draft_workflow=draft, + account=actor, + node_id=node_id, + user_inputs=args.get("inputs") or {}, + query=args.get("query", ""), + files=self._files(draft, raw_files if isinstance(raw_files, list) else None), + ) + return self._execution(execution, include_details=include_details) + + def last_run(self, context: RequestContext, app_id: str, node_id: str) -> dict[str, Any] | None: + try: + app, _, workflow = self._draft(context, app_id, "Workflow not found") + except ValueError as error: + raise WorkflowNotFoundError(str(error)) from error + execution = self._workflows.get_node_last_run(app_model=app, workflow=workflow, node_id=node_id) + return self._execution(execution, include_details=True) if execution is not None else None + + def poll_trigger( + self, context: RequestContext, app_id: str, node_ids: list[str], *, single_node: bool, select_all: bool + ) -> WorkflowTriggerEvent | None: + if single_node: + app, _, workflow = self._draft(context, app_id, "Workflow not found") + snapshot = workflow + else: + app, snapshot = self._reservations.reserve_trigger_debug(context, app_id) + workflow = snapshot + if single_node: + node_id = node_ids[0] + config = workflow.get_node_config_by_id(node_id=node_id) + if not config: + raise ValueError("Node data not found for node %s", node_id) + if workflow.get_node_type_from_node_config(config) == TRIGGER_SCHEDULE_NODE_TYPE: + return WorkflowTriggerEvent(node_id=node_id, workflow_args={}, workflow=snapshot) + try: + if select_all: + event = select_trigger_debug_events( + draft_workflow=workflow, app_model=app, user_id=context.account_id, node_ids=node_ids + ) + else: + event = create_event_poller( + draft_workflow=workflow, + tenant_id=context.active_workspace_id, + user_id=context.account_id, + app_id=app_id, + node_id=node_ids[0], + ).poll() + except Exception as error: + self._cancel_trigger_debug(snapshot, context.account_id) + if isinstance(error, PluginInvokeError): + raise WorkflowTriggerError(error.to_user_friendly_error()) from error + raise + if event is None: + self._cancel_trigger_debug(snapshot, context.account_id) + return None + return WorkflowTriggerEvent(event.node_id, dict(event.workflow_args), snapshot) + + def stop(self, context: RequestContext, app_id: str, task_id: str) -> None: + app = self._definitions.app(context, app_id) + self._tasks.stop_workflow_task( + tenant_id=context.active_workspace_id, + app_id=app_id, + task_id=task_id, + app_mode=app.mode, + owner=None, + ) + + def default_blocks(self) -> Sequence[Mapping[str, object]]: + return self._workflows.get_default_block_configs() + + def default_block(self, block_type: str, filters: dict[str, Any] | None) -> Mapping[str, object]: + return self._workflows.get_default_block_config(node_type=block_type, filters=filters) + + def retire_agents(self, context: RequestContext, agent_ids: list[str]) -> None: + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(self._sessions)).retire_unowned( + tenant_id=context.active_workspace_id, account_id=context.account_id, agent_ids=agent_ids + ) diff --git a/api/services/workflow/variable_contracts.py b/api/services/workflow/variable_contracts.py index bf844aa81bfda8..3bcceb6eff8f19 100644 --- a/api/services/workflow/variable_contracts.py +++ b/api/services/workflow/variable_contracts.py @@ -13,7 +13,7 @@ from graphon.variables.variables import Variable from models import Account, App from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel -from services.workflow.contracts import WorkflowSnapshot +from services.workflow.contracts import WorkflowBindingScope, WorkflowSnapshot def environment_variable_values(workflow: WorkflowSnapshot) -> Sequence[Variable]: @@ -32,9 +32,9 @@ def conversation_variable_values(workflow: WorkflowSnapshot) -> Sequence[Variabl class WorkflowExecutionVariables(Protocol): - def workflow_loader(self, workflow: Workflow, user_id: str) -> VariableLoader: ... + def workflow_loader(self, workflow: WorkflowSnapshot, user_id: str) -> VariableLoader: ... def saver_factory(self, tenant_id: str, account: Account) -> DraftVariableSaverFactory: ... - def get_or_create_conversation(self, account_id: str, app: App, workflow: Workflow) -> str: ... + def get_or_create_conversation(self, account_id: str, app: App, workflow: WorkflowBindingScope) -> str: ... def load_execution_outputs(self, execution: WorkflowNodeExecutionModel) -> Mapping[str, Any] | None: ... diff --git a/api/services/workflow/variable_service.py b/api/services/workflow/variable_service.py index d76ac70fe94730..a6c22e378835cf 100644 --- a/api/services/workflow/variable_service.py +++ b/api/services/workflow/variable_service.py @@ -21,6 +21,7 @@ WorkflowNodeExecutionModel, is_system_variable_editable, ) +from services.workflow.contracts import WorkflowBindingScope, WorkflowSnapshot from services.workflow.draft_variable_service import ( DraftFileRestorer, DraftFileUploader, @@ -32,7 +33,7 @@ VariableResetError, conversation_defaults, ) -from services.workflow.variable_contracts import WorkflowDraftVariableList +from services.workflow.variable_contracts import WorkflowDraftVariableList, conversation_variable_values logger = logging.getLogger(__name__) @@ -149,7 +150,7 @@ def get_or_create_conversation( self, account_id: str, app: App, - workflow: Workflow, + workflow: WorkflowBindingScope, ) -> str: """ get_or_create_conversation creates and returns the ID of a conversation for debugging. @@ -296,8 +297,8 @@ def _loader( file_inputs=self._file_inputs, ) - def workflow_loader(self, workflow: Workflow, user_id: str) -> VariableLoader: - return self._loader(workflow.app_id, workflow.tenant_id, user_id, workflow.conversation_variables) + def workflow_loader(self, workflow: WorkflowSnapshot, user_id: str) -> VariableLoader: + return self._loader(workflow.app_id, workflow.tenant_id, user_id, conversation_variable_values(workflow)) def saver_factory(self, tenant_id: str, account: Account) -> DraftVariableSaverFactory: return partial( diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 4e38563cfa3b1d..1087a381867379 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -1,33 +1,33 @@ import json -from typing import Any, TypedDict - -from sqlalchemy import select -from sqlalchemy.orm import Session - +from collections.abc import Mapping +from copy import deepcopy +from typing import Any, TypedDict, cast + +from core.app.app_config.easy_ui_based_app.dataset.manager import DatasetConfigManager +from core.app.app_config.easy_ui_based_app.model_config.manager import ModelConfigManager +from core.app.app_config.easy_ui_based_app.prompt_template.manager import PromptTemplateConfigManager +from core.app.app_config.easy_ui_based_app.variables.manager import BasicVariablesConfigManager from core.app.app_config.entities import ( + AppAdditionalFeatures, DatasetEntity, DatasetRetrieveConfigEntity, EasyUIBasedAppConfig, + EasyUIBasedAppModelConfigFrom, ExternalDataVariableEntity, ModelConfigEntity, PromptTemplateEntity, ) -from core.app.apps.agent_chat.app_config_manager import AgentChatAppConfigManager -from core.app.apps.chat.app_config_manager import ChatAppConfigManager -from core.app.apps.completion.app_config_manager import CompletionAppConfigManager -from core.helper import encrypter +from core.app.app_config.features.file_upload.manager import FileUploadConfigManager from core.prompt.simple_prompt_transform import SimplePromptTransform from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from events.app_event import app_was_created from graphon.file import FileUploadConfig from graphon.model_runtime.entities.llm_entities import LLMMode from graphon.model_runtime.utils.encoders import jsonable_encoder from graphon.nodes import BuiltinNodeTypes from graphon.variables.input_entities import VariableEntity -from models import Account -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import App, AppMode, AppModelConfig, IconType, load_annotation_reply_config -from models.workflow import Workflow, WorkflowType +from models.api_based_extension import APIBasedExtensionPoint +from models.model import AppMode +from models.workflow_conversion import ConversionExtension, ConvertedWorkflow, WorkflowConversionSource class _NodeType(TypedDict): @@ -52,88 +52,11 @@ class WorkflowConverter: App Convert to Workflow Mode """ - def convert_to_workflow( - self, - app_model: App, - account: Account, - name: str, - icon_type: str, - icon: str, - icon_background: str, - session: Session, - ): - """ - Convert app to workflow - - - basic mode of chatbot app - - - expert mode of chatbot app - - - completion app - - :param app_model: App instance - :param account: Account - :param name: new app name - :param icon: new app icon - :param icon_type: new app icon type - :param icon_background: new app icon background - :return: new App instance - """ - # convert app model config - app_model_config = ( - session.get(AppModelConfig, app_model.app_model_config_id) if app_model.app_model_config_id else None - ) - if not app_model_config: - raise ValueError("App model config is required") - - workflow = self.convert_app_model_config_to_workflow( - app_model=app_model, app_model_config=app_model_config, account_id=account.id, session=session - ) - - # create new app - new_app = App() - new_app.tenant_id = app_model.tenant_id - new_app.name = name or app_model.name + "(workflow)" - new_app.mode = AppMode.ADVANCED_CHAT if app_model.mode == AppMode.CHAT else AppMode.WORKFLOW - new_app.icon_type = IconType(icon_type) if icon_type else app_model.icon_type - new_app.icon = icon or app_model.icon - new_app.icon_background = icon_background or app_model.icon_background - new_app.enable_site = app_model.enable_site - new_app.enable_api = app_model.enable_api - new_app.api_rpm = app_model.api_rpm - new_app.api_rph = app_model.api_rph - new_app.is_demo = False - new_app.is_public = app_model.is_public - new_app.created_by = account.id - new_app.maintainer = account.id - new_app.updated_by = account.id - session.add(new_app) - session.flush() - - workflow.app_id = new_app.id - session.commit() - - app_was_created.send(new_app, account=account, session=session) - session.commit() - - return new_app - - def convert_app_model_config_to_workflow( - self, app_model: App, app_model_config: AppModelConfig, account_id: str, session: Session - ): - """ - Convert app model config to workflow mode - :param app_model: App instance - :param app_model_config: AppModelConfig instance - :param account_id: Account ID - """ - # get new app mode - new_app_mode = self._get_new_app_mode(app_model) - - # convert app model config - app_config = self._convert_to_app_config( - app_model=app_model, app_model_config=app_model_config, session=session - ) + def convert( + self, app_config: EasyUIBasedAppConfig, extensions: Mapping[str, ConversionExtension] + ) -> ConvertedWorkflow: + """Build graph and features from materialized configuration; perform no I/O.""" + new_app_mode = AppMode.WORKFLOW if app_config.app_mode == AppMode.COMPLETION else AppMode.ADVANCED_CHAT # init workflow graph graph: WorkflowGraph = {"nodes": [], "edges": []} @@ -156,10 +79,11 @@ def convert_app_model_config_to_workflow( external_data_variable_node_mapping: dict[str, str] = {} if app_config.external_data_variables: http_request_nodes, external_data_variable_node_mapping = self._convert_to_http_request_node( - app_model=app_model, + app_id=app_config.app_id, + app_mode=app_config.app_mode, variables=app_config.variables, external_data_variables=app_config.external_data_variables, - session=session, + extensions=extensions, ) for http_request_node in http_request_nodes: @@ -176,7 +100,7 @@ def convert_app_model_config_to_workflow( # convert to llm node llm_node = self._convert_to_llm_node( - original_app_mode=AppMode.value_of(app_model.mode), + original_app_mode=app_config.app_mode, new_app_mode=new_app_mode, graph=graph, model_config=app_config.model, @@ -211,70 +135,27 @@ def convert_app_model_config_to_workflow( "sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"), "retriever_resource": app_model_config_dict.get("retriever_resource"), } - case _: - answer_node = self._convert_to_answer_node() - graph = self._append_node(graph, answer_node) - features = { - "text_to_speech": app_model_config_dict.get("text_to_speech"), - "file_upload": app_model_config_dict.get("file_upload"), - "sensitive_word_avoidance": app_model_config_dict.get("sensitive_word_avoidance"), - } - - # create workflow record - workflow = Workflow( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=WorkflowType.from_app_mode(new_app_mode), - version=Workflow.VERSION_DRAFT, - graph=json.dumps(graph), - _features=json.dumps(features), - created_by=account_id, - _environment_variables=[], - _conversation_variables=[], - ) - - session.add(workflow) - session.commit() - - return workflow - - def _convert_to_app_config( - self, app_model: App, app_model_config: AppModelConfig, *, session: Session - ) -> EasyUIBasedAppConfig: - app_mode_enum = AppMode.value_of(app_model.mode) - app_config: EasyUIBasedAppConfig - effective_mode = ( - AppMode.AGENT_CHAT - if app_model.is_agent_with_session(session=session) and app_mode_enum != AppMode.AGENT_CHAT - else app_mode_enum + return ConvertedWorkflow(mode=new_app_mode, graph=cast(dict[str, Any], graph), features=features) + + def app_config(self, source: WorkflowConversionSource) -> EasyUIBasedAppConfig: + """Reuse the config parsers without loading or mutating an ORM App.""" + assert source.model_config is not None + config = deepcopy(source.model_config) + variables, external_variables = BasicVariablesConfigManager.convert(config) + return EasyUIBasedAppConfig( + tenant_id=source.tenant_id, + app_id=source.id, + app_mode=source.mode, + app_model_config_from=EasyUIBasedAppModelConfigFrom.APP_LATEST_CONFIG, + app_model_config_id=source.model_config_id, + app_model_config_dict=cast(dict[str, Any], config), + model=ModelConfigManager.convert(config), + prompt_template=PromptTemplateConfigManager.convert(config), + dataset=DatasetConfigManager.convert(config), + variables=variables, + external_data_variables=external_variables, + additional_features=AppAdditionalFeatures(file_upload=FileUploadConfigManager.convert(config)), ) - match effective_mode: - case AppMode.AGENT_CHAT: - app_model.mode = AppMode.AGENT_CHAT - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = AgentChatAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case AppMode.CHAT: - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = ChatAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case AppMode.COMPLETION: - annotation_reply = load_annotation_reply_config(session, app_model_config.app_id) - app_config = CompletionAppConfigManager.get_app_config( - app_model=app_model, - app_model_config=app_model_config, - annotation_reply=annotation_reply, - ) - case _: - raise ValueError("Invalid app mode") - - return app_config def _convert_to_start_node(self, variables: list[VariableEntity]) -> _NodeType: """ @@ -294,14 +175,16 @@ def _convert_to_start_node(self, variables: list[VariableEntity]) -> _NodeType: def _convert_to_http_request_node( self, - app_model: App, + app_id: str, + app_mode: AppMode, variables: list[VariableEntity], external_data_variables: list[ExternalDataVariableEntity], - session: Session, + extensions: Mapping[str, ConversionExtension], ) -> tuple[list[_NodeType], dict[str, str]]: """ Convert API Based Extension to HTTP Request Node - :param app_model: App instance + :param app_id: source app id + :param app_mode: source app mode :param variables: list of variables :param external_data_variables: list of external data variables :return: @@ -309,7 +192,6 @@ def _convert_to_http_request_node( index = 1 nodes = [] external_data_variable_node_mapping = {} - tenant_id = app_model.tenant_id for external_data_variable in external_data_variables: tool_type = external_data_variable.type if tool_type != "api": @@ -323,13 +205,8 @@ def _convert_to_http_request_node( if not api_based_extension_id: continue - # get api_based_extension - api_based_extension = self._get_api_based_extension( - tenant_id=tenant_id, api_based_extension_id=api_based_extension_id, session=session - ) - - # decrypt api_key - api_key = encrypter.decrypt_token(tenant_id=tenant_id, token=api_based_extension.api_key) + api_based_extension = extensions[api_based_extension_id] + api_key = api_based_extension.api_key inputs = {} for v in variables: @@ -338,10 +215,10 @@ def _convert_to_http_request_node( request_body = { "point": APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY, "params": { - "app_id": app_model.id, + "app_id": app_id, "tool_variable": tool_variable, "inputs": inputs, - "query": "{{#sys.query#}}" if app_model.mode == AppMode.CHAT else "", + "query": "{{#sys.query#}}" if app_mode == AppMode.CHAT else "", }, } @@ -669,32 +546,3 @@ def _append_node(self, graph: WorkflowGraph, node: _NodeType): graph["nodes"].append(node) graph["edges"].append(self._create_edge(previous_node["id"], node["id"])) return graph - - def _get_new_app_mode(self, app_model: App) -> AppMode: - """ - Get new app mode - :param app_model: App instance - :return: AppMode - """ - if app_model.mode == AppMode.COMPLETION: - return AppMode.WORKFLOW - else: - return AppMode.ADVANCED_CHAT - - def _get_api_based_extension(self, tenant_id: str, api_based_extension_id: str, session: Session): - """ - Get API Based Extension - :param tenant_id: tenant id - :param api_based_extension_id: api based extension id - :return: - """ - api_based_extension = session.scalar( - select(APIBasedExtension) - .where(APIBasedExtension.tenant_id == tenant_id, APIBasedExtension.id == api_based_extension_id) - .limit(1) - ) - - if not api_based_extension: - raise ValueError(f"API Based Extension not found, id: {api_based_extension_id}") - - return api_based_extension diff --git a/api/services/workflow_draft_variable_service.py b/api/services/workflow_draft_variable_service.py index 4971305c1275f5..e428e109a226ad 100644 --- a/api/services/workflow_draft_variable_service.py +++ b/api/services/workflow_draft_variable_service.py @@ -42,7 +42,7 @@ from libs.datetime_utils import naive_utc_now from libs.uuid_utils import uuidv7 from models import Account, App, Conversation -from models.enums import ConversationFromSource, ConversationStatus, DraftVariableType, InvokeFrom +from models.enums import ConversationFromSource, ConversationStatus, DraftVariableType, ExecutionOffLoadType, InvokeFrom from models.utils.file_input_compat import build_file_from_stored_mapping from models.workflow import Workflow, WorkflowDraftVariable, WorkflowDraftVariableFile, is_system_variable_editable from repositories.factory import DifyAPIRepositoryFactory @@ -399,8 +399,11 @@ def _reset_node_var_or_sys_var( logger.warning("draft variable has no node_execution_id, id=%s, name=%s", variable.id, variable.name) return None - node_exec = self._api_node_execution_repo.get_execution_by_id(variable.node_execution_id) - if node_exec is None: + node_exec = self._api_node_execution_repo.get_execution_by_id( + variable.node_execution_id, + tenant_id=workflow.tenant_id, + ) + if node_exec is None or node_exec.app_id != workflow.app_id: logger.warning( "Node execution not found for draft variable, id=%s, name=%s, node_execution_id=%s", variable.id, @@ -411,7 +414,21 @@ def _reset_node_var_or_sys_var( self._session.flush() return None - outputs_dict = node_exec.load_full_outputs(self._session, storage) or {} + output_offload = next( + (item for item in node_exec.offload_data if item.type_ == ExecutionOffLoadType.OUTPUTS), + None, + ) + if output_offload is None: + outputs_dict = node_exec.outputs_dict or {} + else: + if ( + output_offload.tenant_id != node_exec.tenant_id + or output_offload.app_id != node_exec.app_id + or output_offload.file is None + or output_offload.file.tenant_id != node_exec.tenant_id + ): + raise VariableResetError(f"Execution output file not found, execution_id={node_exec.id}") + outputs_dict = json.loads(storage.load(output_offload.file.key)) or {} # a sentinel value used to check the absent of the output variable key. absent = object() diff --git a/api/services/workflow_node_execution_trace_service.py b/api/services/workflow_node_execution_trace_service.py index f016afa80ea1f0..37d9380351b4d9 100644 --- a/api/services/workflow_node_execution_trace_service.py +++ b/api/services/workflow_node_execution_trace_service.py @@ -14,7 +14,9 @@ from libs.helper import to_timestamp from models.workflow import WorkflowNodeExecutionModel from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository +from services.tools.provider_queries import ToolProviderIcons from services.variable_truncator import VariableTruncator +from services.workflow.node_execution_extras import node_execution_extras logger = logging.getLogger(__name__) @@ -61,11 +63,12 @@ def assemble_workflow_node_execution_traces( repository: DifyAPIWorkflowNodeExecutionRepository, *, session: Session, + tool_providers: ToolProviderIcons, ) -> list[WorkflowNodeExecutionTrace]: """Expand valid persisted retry attempts before each terminal execution.""" traces: list[WorkflowNodeExecutionTrace] = [] for execution in executions: - traces.extend(_expand_execution(execution, repository, session=session)) + traces.extend(_expand_execution(execution, repository, session=session, tool_providers=tool_providers)) return traces @@ -74,12 +77,16 @@ def _expand_execution( repository: DifyAPIWorkflowNodeExecutionRepository, *, session: Session, + tool_providers: ToolProviderIcons, ) -> list[WorkflowNodeExecutionTrace]: full_process_data = _load_full_process_data(execution, repository) retry_attempts = _parse_retry_attempts(execution, full_process_data) terminal_metadata = execution.execution_metadata_dict - traces = [_retry_trace(execution, attempt, terminal_metadata, session=session) for attempt in retry_attempts] - traces.append(_terminal_trace(execution, session=session)) + traces = [ + _retry_trace(execution, attempt, terminal_metadata, session=session, tool_providers=tool_providers) + for attempt in retry_attempts + ] + traces.append(_terminal_trace(execution, session=session, tool_providers=tool_providers)) return traces @@ -140,6 +147,7 @@ def _retry_trace( terminal_metadata: Mapping[str, Any], *, session: Session, + tool_providers: ToolProviderIcons, ) -> WorkflowNodeExecutionTrace: truncator = VariableTruncator.default() inputs, inputs_truncated = truncator.truncate_variable_mapping(attempt.inputs) @@ -166,7 +174,7 @@ def _retry_trace( error=attempt.error, elapsed_time=attempt.elapsed_time, execution_metadata=execution_metadata, - extras=execution.extras, + extras=node_execution_extras(execution, tool_providers=tool_providers), created_at=attempt.created_at, created_by_role=_enum_value(execution.created_by_role), created_by=execution.created_by, @@ -180,7 +188,9 @@ def _retry_trace( ) -def _terminal_trace(execution: WorkflowNodeExecutionModel, *, session: Session) -> WorkflowNodeExecutionTrace: +def _terminal_trace( + execution: WorkflowNodeExecutionModel, *, session: Session, tool_providers: ToolProviderIcons +) -> WorkflowNodeExecutionTrace: process_data = execution.process_data_dict if process_data is not None and RETRY_HISTORY_PROCESS_DATA_KEY in process_data: process_data = dict(process_data) @@ -205,7 +215,7 @@ def _terminal_trace(execution: WorkflowNodeExecutionModel, *, session: Session) error=execution.error, elapsed_time=execution.elapsed_time, execution_metadata=execution.execution_metadata_dict, - extras=execution.extras, + extras=node_execution_extras(execution, tool_providers=tool_providers), created_at=to_timestamp(execution.created_at), created_by_role=_enum_value(execution.created_by_role), created_by=execution.created_by, diff --git a/api/services/workflow_run_agg.py b/api/services/workflow_run_agg.py index b541306f97a603..c2177f73f3d680 100644 --- a/api/services/workflow_run_agg.py +++ b/api/services/workflow_run_agg.py @@ -20,8 +20,6 @@ from core.db.session_factory import session_factory from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository from core.tools.workflow_as_tool.repository import WorkflowToolSource -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import resolve_dify_run_context from core.workflow.nodes.agent_v2.workspace_retirement_layer import build_workflow_agent_workspace_retirement_layer from core.workflow.nodes.human_input.pause_reason import PauseReason from core.workflow.nodes.human_input.session_binding import default_session_binding @@ -56,6 +54,8 @@ from repositories.sqlalchemy_api_workflow_run_repository import WorkflowPauseStoppedError from repositories.workflow_tool_source_repository import SQLAlchemyWorkflowToolSourceRepository from services.workflow.execution.adapters.human_input_events import build_human_input_pause_reason +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.adapters.node_runtime import resolve_dify_run_context from services.workflow.execution.contracts import PreparedWorkflowRun, WorkflowRunRunner from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer from services.workflow_run_index import WorkflowRunIndex @@ -95,8 +95,18 @@ def __init__( if _WORKFLOW_TOOL_SNAPSHOT_KEY in extras else None ) + node_factory = self._entry.graph_engine.graph.node_factory + workflow_runtime = node_factory.workflow_runtime if isinstance(node_factory, DifyNodeFactory) else None + draft_variable_saver = node_factory.draft_variable_saver if isinstance(node_factory, DifyNodeFactory) else None + tool_source_repository = SQLAlchemyWorkflowToolSourceRepository( + session_factory.get_session_maker(), + workflow_runtime=workflow_runtime, + draft_variable_saver=draft_variable_saver, + ) materialized_tools = WorkflowToolSourceService( - SQLAlchemyWorkflowToolSourceRepository(session_factory.get_session_maker()) + tool_source_repository, + workflow_runtime=workflow_runtime, + draft_variable_saver=draft_variable_saver, ).load( tenant_id=self._run_context.tenant_id, graph_config=self._entry.graph_config, @@ -120,7 +130,6 @@ def __init__( # The generate entity is serialized by both durable pause/resume paths. # Capture every reachable provider before any node can pause or republish it. extras[_WORKFLOW_TOOL_SNAPSHOT_KEY] = materialized_tools.snapshot().model_dump(mode="json") - node_factory = self._entry.graph_engine.graph.node_factory if isinstance(node_factory, DifyNodeFactory): node_factory.workflow_tools.update(materialized_tools.tools) self._human_input_v2_forms: dict[str, PreparedForm] = {} diff --git a/api/services/workflow_run_service.py b/api/services/workflow_run_service.py index 399fd6647cdf97..6005f2dac48d54 100644 --- a/api/services/workflow_run_service.py +++ b/api/services/workflow_run_service.py @@ -21,6 +21,7 @@ ) from repositories.api_workflow_node_execution_repository import DifyAPIWorkflowNodeExecutionRepository from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from services.tools.provider_queries import ToolProviderIcons from services.workflow_node_execution_trace_service import ( WorkflowNodeExecutionTrace, assemble_workflow_node_execution_traces, @@ -57,10 +58,12 @@ def __init__( *, workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, node_executions: DifyAPIWorkflowNodeExecutionRepository, + tool_providers: ToolProviderIcons, session_factory: sessionmaker[Session], ) -> None: self._workflow_runs = workflow_runs self._node_executions = node_executions + self._tool_providers = tool_providers self._session_factory = session_factory def get_paginate_advanced_chat_workflow_runs( @@ -213,7 +216,9 @@ def get_workflow_run_node_executions( app_id=app_id, workflow_run_id=run_id, ) - return assemble_workflow_node_execution_traces(node_executions, self._node_executions, session=db.session()) + return assemble_workflow_node_execution_traces( + node_executions, self._node_executions, session=db.session(), tool_providers=self._tool_providers + ) def get_workflow_tool_node_executions( self, @@ -233,7 +238,12 @@ def get_workflow_tool_node_executions( workflow_run_id=run_id, parent_node_execution_id=node_execution_id, ) - traces = assemble_workflow_node_execution_traces(executions, self._node_executions, session=db.session()) + traces = assemble_workflow_node_execution_traces( + executions, + self._node_executions, + session=db.session(), + tool_providers=self._tool_providers, + ) for trace in traces: if trace.process_data is not None: trace.process_data = { diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 5d73ae8464dcfb..f0790ae21adc2a 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -1,24 +1,16 @@ -import json -import logging import time import uuid from collections.abc import Callable, Generator, Mapping, Sequence -from dataclasses import dataclass -from typing import Any, cast +from typing import Any, Protocol, cast -from sqlalchemy import exists, inspect, select from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager -from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager from core.app.file_access import DatabaseFileAccessController from core.entities import PluginCredentialType from core.plugin.impl.model_runtime_factory import create_plugin_model_assembly, create_plugin_provider_manager -from core.repositories import DifyCoreRepositoryFactory -from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepositoryImpl from core.trigger.constants import is_trigger_node_type -from core.workflow.human_input_adapter import adapt_human_input_node_data_for_graph, parse_human_input_delivery_methods +from core.workflow.human_input_adapter import adapt_human_input_node_data_for_graph from core.workflow.llm_environment_variable import ( LLMEnvironmentVariable, parse_llm_model_selector, @@ -26,40 +18,20 @@ should_resolve_llm_model_selector, validate_llm_environment_model_references, ) -from core.workflow.node_factory import ( - LATEST_VERSION, - get_node_type_classes_mapping, - is_start_node_type, -) -from core.workflow.node_runtime import ( - apply_dify_debug_email_recipient, -) -from core.workflow.nodes.agent_v2.discriminator import is_dify_agent_node_data -from core.workflow.nodes.human_input.callback import ( - DifyHITLCallback, - render_form_content_before_submission, - resolve_default_values, -) -from core.workflow.nodes.human_input.pause_reason import HumanInputRequired from core.workflow.system_variables import ( SystemVariableKey, build_bootstrap_variables, build_system_variables, default_system_variables, - system_variable_selector, + get_system_text, ) from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_pool -from core.workflow.workflow_entry import WorkflowEntry from enterprise.telemetry.draft_trace import enqueue_draft_node_execution_trace from enums import CloudPlan, DeploymentEdition -from enums.human_input import HumanInputFormKind, RecipientType -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated from extensions.ext_database import db -from extensions.ext_storage import storage from factories.file_factory import build_from_mapping, build_from_mappings from graphon.engine_events import NodeEvent, NodeRunFailedEvent, NodeRunStartedEvent, NodeRunSucceededEvent from graphon.entities import WorkflowNodeExecution -from graphon.entities.graph_config import NodeConfigDict from graphon.enums import ( ErrorStrategy, NodeType, @@ -76,136 +48,40 @@ from graphon.nodes.llm.entities import ModelConfig from graphon.nodes.start.entities import StartNodeData from graphon.runtime import VariablePool -from graphon.variable_loader import load_into_variable_pool from graphon.variables import VariableBase from graphon.variables.input_entities import VariableEntityType from graphon.variables.variables import Variable from libs.datetime_utils import naive_utc_now, to_utc_timestamp from models import Account -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding -from models.human_input import HumanInputFormRecipient -from models.human_input_delivery import DeliveryChannelConfig -from models.human_input_entities import FormInputConfig, HumanInputNodeData +from models.human_input_entities import HumanInputNodeData from models.model import App, AppMode -from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType from repositories.factory import DifyAPIRepositoryFactory -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService +from repositories.workflow.definition_repository import WorkflowDefinitionStore from services.billing_service import BillingService -from services.enterprise.enterprise_service import EnterpriseService +from services.credentials.query import CredentialQuery from services.errors.app import ( - IsDraftWorkflowError, TriggerNodeLimitExceededError, - WorkflowHashNotEqualError, - WorkflowNotFoundError, ) -from services.human_input_service import HumanInputService from services.system_feature_service import SystemFeatureService -from services.workflow.workflow_converter import WorkflowConverter -from services.workflow_node_variables import get_human_input_form_variable_mapping -from services.workflow_ref_service import WorkflowRef -from services.workflow_run_agg import WorkflowRunAgg -from services.workflow_version_number_service import allocate_version_number - -from .errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError -from .human_input_delivery_test_service import ( - DeliveryTestContext, - DeliveryTestEmailRecipient, - DeliveryTestError, - DeliveryTestUnsupportedError, - HumanInputDeliveryTestService, +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.chatflow.app_config_manager import AdvancedChatAppConfigManager +from services.workflow.execution.adapters.node_factory import ( + LATEST_VERSION, + get_node_type_classes_mapping, + is_start_node_type, ) -from .workflow_draft_variable_service import DraftVariableSaver, DraftVarLoader, WorkflowDraftVariableService -from .workflow_restore import apply_published_workflow_snapshot_to_draft - - -@dataclass(frozen=True) -class _DebugHumanInputNode: - node_id: str - title: str - node_data: HumanInputNodeData - variable_pool: VariablePool - - def render_form_content_before_submission(self) -> str: - return render_form_content_before_submission( - self.node_data, - variable_pool=self.variable_pool, - ) - - def resolve_default_values(self) -> Mapping[str, Any]: - return resolve_default_values( - self.node_data, - variable_pool=self.variable_pool, - ) - - def render_form_content_with_outputs( - self, - form_content: str, - outputs: Mapping[str, Any], - field_names: Sequence[str], - form_inputs: Sequence[FormInputConfig] | None = None, - ) -> str: - return DifyHITLCallback.render_form_content_with_outputs( - form_content=form_content, - outputs=outputs, # type: ignore[arg-type] - field_names=field_names, - form_inputs=form_inputs, - ) - - @property - def workflow_execution_id(self) -> str: - return "debug-human-input" - - @property - def node_title(self) -> str: - return self.title - - -HumanInputNode = _DebugHumanInputNode +from services.workflow.execution.adapters.workflow.app_config_manager import WorkflowAppConfigManager +from services.workflow.execution.adapters.workflow_entry import WorkflowEntry +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables, environment_variable_values _file_access_controller = DatabaseFileAccessController() -def _merge_environment_variable_patch( - current_variables: Sequence[VariableBase], - environment_variable_upserts: Sequence[VariableBase], - deleted_environment_variable_ids: Sequence[str], -) -> list[VariableBase]: - """Merge a per-ID environment-variable patch while preserving untouched server values.""" - upserts_by_id: dict[str, VariableBase] = {} - for variable in environment_variable_upserts: - if not variable.id: - raise ValueError("Patched environment variables require an id.") - if variable.id in upserts_by_id: - raise ValueError(f"Duplicate patched environment variable id: {variable.id}") - upserts_by_id[variable.id] = variable - - deleted_ids = set(deleted_environment_variable_ids) - if len(deleted_ids) != len(deleted_environment_variable_ids): - raise ValueError("Deleted environment variable ids must be unique.") - if "" in deleted_ids: - raise ValueError("Deleted environment variable ids must not be empty.") - if conflicting_ids := deleted_ids.intersection(upserts_by_id): - conflicting_id = min(conflicting_ids) - raise ValueError(f"Environment variable cannot be upserted and deleted in the same patch: {conflicting_id}") - - existing_ids: set[str] = set() - merged_variables: list[VariableBase] = [] - for variable in current_variables: - variable_id = variable.id - if variable_id: - existing_ids.add(variable_id) - if variable_id in deleted_ids: - continue - merged_variables.append(upserts_by_id.get(variable_id, variable)) - - merged_variables.extend( - variable for variable_id, variable in upserts_by_id.items() if variable_id not in existing_ids - ) - names = [variable.name for variable in merged_variables] - if len(set(names)) != len(names): - raise ValueError("Environment variable names must be unique.") - return merged_variables +class WorkflowIdentity(Protocol): + @property + def id(self) -> str: ... class WorkflowService: @@ -213,15 +89,24 @@ class WorkflowService: Workflow Service """ - def __init__(self, session_maker: sessionmaker | None = None): + def __init__( + self, + session_maker: sessionmaker | None = None, + *, + runtime: WorkflowRuntime | None = None, + ): """Initialize WorkflowService with repository dependencies.""" if session_maker is None: session_maker = sessionmaker(bind=db.engine, expire_on_commit=False) + self._runtime = runtime + self._sessions = session_maker self._node_execution_service_repo = DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker ) - def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> WorkflowNodeExecutionModel | None: + def get_node_last_run( + self, app_model: App, workflow: WorkflowIdentity, node_id: str + ) -> WorkflowNodeExecutionModel | None: """ Get the most recent execution for a specific node. @@ -240,469 +125,28 @@ def get_node_last_run(self, app_model: App, workflow: Workflow, node_id: str) -> node_id=node_id, ) - def is_workflow_exist(self, app_model: App, *, session: Session) -> bool: - stmt = select( - exists().where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - ) - return session.execute(stmt).scalar_one() - - @staticmethod - def get_draft_workflow(app_model: App, *, session: Session) -> Workflow | None: - """ - Return the app's draft workflow, or None if no draft exists. - - Reuses the caller's active session so workflow reads stay in the same - transaction as the surrounding request or task. No service construction - or Flask database binding is needed for this read. - """ - return session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - .limit(1) - ) - - def get_draft_workflow_for_update(self, app_model: App, *, session: Session) -> Workflow | None: - """Return the app draft while holding its row lock for the caller's transaction.""" - return session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - .limit(1) - .with_for_update() - ) + def get_draft_workflow( + self, app_model: App, workflow_id: str | None = None, *, session: Session + ) -> Workflow | None: + return WorkflowDefinitionStore.get_draft_workflow(app_model, workflow_id, session=session) - @staticmethod def get_published_workflow_by_id( + self, app_model: App, workflow_id: str, *, session: Session, for_update: bool = False, ) -> Workflow | None: - """Fetch a published workflow by ID in the caller's transaction. - - With ``for_update=True``, the source version stays locked until that - transaction ends. Restore uses the lock while copying Agent bindings so - a concurrent delete cannot release the same owner. - """ - stmt = ( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.id == workflow_id, - ) - .limit(1) + return WorkflowDefinitionStore.get_published_workflow_by_id( + app_model, workflow_id, session=session, for_update=for_update ) - if for_update: - stmt = stmt.with_for_update() - workflow = session.scalar(stmt) - if not workflow: - return None - if workflow.version == Workflow.VERSION_DRAFT: - raise IsDraftWorkflowError( - f"Cannot use draft workflow version. Workflow ID: {workflow_id}. " - f"Please use a published workflow version or leave workflow_id empty." - ) - return workflow def get_published_workflow(self, app_model: App, *, session: Session) -> Workflow | None: - """ - Get published workflow - - Reuses the caller's active session so workflow reads stay in the same - transaction as the surrounding request or task. - """ - - if not app_model.workflow_id: - return None - - workflow = session.scalar( - select(Workflow) - .where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.id == app_model.workflow_id, - ) - .limit(1) - ) - - return workflow - - def get_tenant_app_maintainers( - self, app_ids: Sequence[str], tenant_id: str, *, session: Session - ) -> dict[str, str | None]: - """Return requested normal apps and their maintainers within a tenant.""" - if not app_ids: - return {} - - stmt = select(App.id, App.maintainer).where( - App.id.in_(app_ids), App.tenant_id == tenant_id, App.status == "normal" - ) - return {str(app_id): maintainer for app_id, maintainer in session.execute(stmt)} - - def get_all_published_workflow( - self, - *, - session: Session, - app_model: App, - page: int, - limit: int, - user_id: str | None, - named_only: bool = False, - include_draft: bool = True, - ) -> tuple[Sequence[Workflow], bool]: - """ - Get published workflow with pagination - - The console lists the draft alongside published versions (``include_draft=True``, - the default); callers that only want published versions, where the draft would - break restore or ordering, pass ``include_draft=False``. - """ - if not app_model.workflow_id: - return [], False - - stmt = ( - select(Workflow) - .where(Workflow.app_id == app_model.id) - # The draft leads the list; its `created_at` is the app's creation time, so it would - # otherwise sort last. Published versions then order by publish time: `version` is a - # stringified timestamp whose microseconds are omitted when zero, so ordering by it - # misplaces versions across second boundaries, and `version_number` is NULL for - # versions published before numbering was introduced. - .order_by( - (Workflow.version == Workflow.VERSION_DRAFT).desc(), - Workflow.created_at.desc(), - Workflow.id.desc(), - ) - .limit(limit + 1) - .offset((page - 1) * limit) - ) - - if user_id: - stmt = stmt.where(Workflow.created_by == user_id) - - if named_only: - stmt = stmt.where(Workflow.marked_name != "") - - if not include_draft: - stmt = stmt.where(Workflow.version != Workflow.VERSION_DRAFT) - - workflows = session.scalars(stmt).all() - - has_more = len(workflows) > limit - if has_more: - workflows = workflows[:-1] - - return workflows, has_more - - def sync_draft_workflow( - self, - *, - app_model: App, - graph: dict[str, Any], - features: dict[str, Any], - unique_hash: str | None, - account: Account, - environment_variables: Sequence[VariableBase], - conversation_variables: Sequence[VariableBase], - session: Session, - environment_variable_upserts: Sequence[VariableBase] | None = None, - deleted_environment_variable_ids: Sequence[str] = (), - preserve_environment_variables: bool = False, - commit: bool = True, - sync_agent_bindings: bool = True, - graph_only: bool = False, - ) -> Workflow: - """ - Sync draft workflow. - - DSL import disables the intermediate commit and Agent binding sync so - portable package references can be materialized atomically after the - draft workflow has received its target-workspace id. - - Existing drafts are row-locked before the hash check. Collaborative - graph-only saves preserve independently persisted draft fields, while - an explicit per-ID environment patch is merged with the graph. - - :raises WorkflowHashNotEqualError - """ - if environment_variable_upserts is None and deleted_environment_variable_ids: - raise ValueError("Deleted environment variable ids require an environment variable patch.") - - # fetch draft workflow by app_model - workflow = self.get_draft_workflow_for_update(app_model=app_model, session=session) - - if workflow and workflow.unique_hash != unique_hash: - raise WorkflowHashNotEqualError() - - # Collaboration persists features and variables through dedicated endpoints. A graph save - # must not overwrite those newer database values with another collaborator's stale cache. - if not graph_only or not workflow: - self.validate_features_structure(app_model=app_model, features=features) - - # validate graph structure - self.validate_graph_structure(graph=graph) - - # create draft workflow if not found - if not workflow: - initial_environment_variables = ( - _merge_environment_variable_patch( - environment_variables, - environment_variable_upserts, - deleted_environment_variable_ids, - ) - if environment_variable_upserts is not None - else list(environment_variables) - ) - workflow = Workflow( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=WorkflowType.from_app_mode(app_model.mode), - version=Workflow.VERSION_DRAFT, - graph=json.dumps(graph), - _features=json.dumps(features), - created_by=account.id, - _environment_variables=initial_environment_variables, - _conversation_variables=conversation_variables, - ) - session.add(workflow) - # update draft workflow if found - else: - workflow.graph = json.dumps(graph) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - if not graph_only: - workflow.features = json.dumps(features) - workflow.conversation_variables = conversation_variables - if environment_variable_upserts is not None: - workflow.environment_variables = _merge_environment_variable_patch( - workflow.environment_variables, - environment_variable_upserts, - deleted_environment_variable_ids, - ) - elif not graph_only and not preserve_environment_variables: - workflow.environment_variables = environment_variables - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - session.flush() - retirement_candidates: set[str] = set() - if sync_agent_bindings: - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, - account_id=account.id, - ) - WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync( - session=session, - draft_workflow=workflow, - ) - - # commit db session changes - if commit: - session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - - # trigger app workflow events - if commit: - app_draft_workflow_was_synced.send(app_model, synced_draft_workflow=workflow) - - # return draft workflow - return workflow - - def update_draft_workflow_environment_variables( - self, - *, - app_model: App, - environment_variables: Sequence[VariableBase], - account: Account, - session: Session, - ) -> None: - """Replace every environment variable on a draft workflow and commit the transaction.""" - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.environment_variables = environment_variables - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def patch_draft_workflow_environment_variables( - self, - *, - app_model: App, - environment_variables: Sequence[VariableBase], - deleted_environment_variable_ids: Sequence[str], - account: Account, - session: Session, - ) -> None: - """Atomically merge per-ID environment-variable upserts and deletions into a draft workflow. - - The draft row is locked before reading its current variables so concurrent patches preserve - variables they do not touch. Existing variables keep their order and new variables are appended. - The transaction is committed before this method returns. - """ - workflow = self.get_draft_workflow_for_update(app_model=app_model, session=session) - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.environment_variables = _merge_environment_variable_patch( - workflow.environment_variables, - environment_variables, - deleted_environment_variable_ids, - ) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - session.commit() - - def update_draft_workflow_conversation_variables( - self, - *, - app_model: App, - conversation_variables: Sequence[VariableBase], - account: Account, - session: Session, - ): - """ - Update draft workflow conversation variables - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - workflow.conversation_variables = conversation_variables - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def update_draft_workflow_features( - self, - *, - app_model: App, - features: dict, - account: Account, - session: Session, - ): - """ - Update draft workflow features - """ - # fetch draft workflow by app_model - workflow = self.get_draft_workflow(app_model=app_model, session=session) - - if not workflow: - raise ValueError("No draft workflow found.") - - # validate features structure - self.validate_features_structure(app_model=app_model, features=features) - - workflow.features = json.dumps(features) - workflow.updated_by = account.id - workflow.updated_at = naive_utc_now() - - # commit db session changes - session.commit() - - def restore_published_workflow_to_draft( - self, - *, - app_model: App, - workflow_id: str, - account: Account, - session: Session, - ) -> Workflow: - """Restore a published workflow snapshot into the draft workflow. - - Secret environment variables are copied server-side from the selected - published workflow so the normal draft sync flow stays stateless. - """ - source_workflow = self.get_published_workflow_by_id( - app_model=app_model, - workflow_id=workflow_id, - session=session, - for_update=True, - ) - if not source_workflow: - raise WorkflowNotFoundError("Workflow not found.") - - self.validate_features_structure(app_model=app_model, features=source_workflow.normalized_features_dict) - self.validate_graph_structure(graph=source_workflow.graph_dict) - - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - draft_workflow, is_new_draft = apply_published_workflow_snapshot_to_draft( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account=account, - updated_at_factory=naive_utc_now, - ) - - if is_new_draft: - session.add(draft_workflow) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - session.flush() - retirement_candidates = WorkflowAgentPublishService.restore_agent_node_bindings_to_draft( - session=session, - source_workflow=source_workflow, - draft_workflow=draft_workflow, - account_id=account.id, - ) - - session.commit() - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app_model.tenant_id, - agent_ids=retirement_candidates, - account_id=account.id, - ) - app_draft_workflow_was_synced.send(app_model, synced_draft_workflow=draft_workflow) - - return draft_workflow - - def publish_workflow( - self, - *, - session: Session, - app_model: App, - account: Account, - marked_name: str = "", - marked_comment: str = "", - ) -> Workflow: - draft_workflow_stmt = select(Workflow).where( - Workflow.tenant_id == app_model.tenant_id, - Workflow.app_id == app_model.id, - Workflow.version == Workflow.VERSION_DRAFT, - ) - draft_workflow = session.scalar(draft_workflow_stmt) - if not draft_workflow: - raise ValueError("No valid workflow found.") + return WorkflowDefinitionStore.get_published_workflow(app_model, session=session) + def validate_publication(self, app_model: App, draft_workflow: Workflow, *, credentials: CredentialQuery) -> None: + """Validate an already loaded revision without changing workflow state.""" validate_llm_environment_model_references( graph=draft_workflow.graph_dict, environment_variables=draft_workflow.environment_variables, @@ -710,26 +154,21 @@ def publish_workflow( # Validate credentials before publishing, for credential policy check if SystemFeatureService.is_plugin_manager_enabled(): - self._validate_workflow_credentials(draft_workflow, session=session) + self._validate_workflow_credentials(draft_workflow, credentials=credentials) # validate graph structure self.validate_graph_structure(graph=draft_workflow.graph_dict) + from services.knowledge_fs.knowledge_retrieval_v2.validation import ( validate_control_space_references, ) - validate_control_space_references( - session=session, - tenant_id=app_model.tenant_id, - graph=draft_workflow.graph_dict, - ) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=draft_workflow, - ) + with self._sessions() as session: + validate_control_space_references( + session=session, + tenant_id=app_model.tenant_id, + graph=draft_workflow.graph_dict, + ) # billing check if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD: @@ -746,42 +185,7 @@ def publish_workflow( if trigger_node_count > 2: raise TriggerNodeLimitExceededError(count=trigger_node_count, limit=2) - # create new workflow - workflow = Workflow.new( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=draft_workflow.type, - version=Workflow.version_from_datetime(naive_utc_now()), - version_number=allocate_version_number(session=session, app_id=app_model.id), - graph=draft_workflow.graph, - created_by=account.id, - environment_variables=draft_workflow.environment_variables, - conversation_variables=draft_workflow.conversation_variables, - marked_name=marked_name, - marked_comment=marked_comment, - rag_pipeline_variables=draft_workflow.rag_pipeline_variables, - features=draft_workflow.features, - ) - - # commit db session changes - session.add(workflow) - WorkflowAgentPublishService.copy_agent_node_bindings_to_published( - session=session, - draft_workflow=draft_workflow, - published_workflow=workflow, - ) - - # trigger app workflow events - app_published_workflow_was_updated.send( - app_model, - published_workflow=workflow, - session=session, - ) - - # return new workflow - return workflow - - def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session) -> None: + def _validate_workflow_credentials(self, workflow: Workflow, *, credentials: CredentialQuery) -> None: """ Validate all credentials in workflow nodes before publishing. @@ -800,7 +204,7 @@ def _validate_workflow_credentials(self, workflow: Workflow, *, session: Session ) for node in nodes: - self.validate_node_credentials(workflow.tenant_id, node, environment_variables, session=session) + self.validate_node_credentials(workflow.tenant_id, node, environment_variables, credentials=credentials) def validate_node_credentials( self, @@ -808,7 +212,7 @@ def validate_node_credentials( node: Mapping[str, Any], environment_variables: Mapping[str, VariableBase], *, - session: Session, + credentials: CredentialQuery, ) -> None: """Publish's credential check for one node. @@ -835,7 +239,7 @@ def validate_node_credentials( ) else: # Check default workspace credential for this provider - self._check_default_tool_credential(tenant_id, provider, session=session) + self._check_default_tool_credential(tenant_id, provider, credentials=credentials) elif node_type == "agent": agent_params = node_data.get("agent_parameters", {}) @@ -847,7 +251,7 @@ def validate_node_credentials( # Validate load balancing credentials for agent model if load balancing is enabled agent_model_node_data = {"model": model_config} self._validate_load_balancing_credentials( - tenant_id, agent_model_node_data, node_id, session=session + tenant_id, agent_model_node_data, node_id, credentials=credentials ) # Validate agent tools @@ -862,7 +266,7 @@ def validate_node_credentials( check_credential_policy_compliance(credential_id, provider, PluginCredentialType.TOOL) else: - self._check_default_tool_credential(tenant_id, provider, session=session) + self._check_default_tool_credential(tenant_id, provider, credentials=credentials) elif node_type in ["llm", "knowledge_retrieval", "parameter_extractor", "question_classifier"]: validation_node_data = node_data @@ -888,7 +292,9 @@ def validate_node_credentials( # Validate that the provider+model combination can fetch valid credentials self._validate_llm_model_config(tenant_id, provider, model_name) # Validate load balancing credentials if load balancing is enabled - self._validate_load_balancing_credentials(tenant_id, validation_node_data, node_id, session=session) + self._validate_load_balancing_credentials( + tenant_id, validation_node_data, node_id, credentials=credentials + ) else: raise ValueError(f"Node {node_id} ({node_type}): Missing provider or model configuration") @@ -951,7 +357,7 @@ def _validate_llm_model_config(self, tenant_id: str, provider: str, model_name: f"Failed to validate LLM model configuration (provider: {provider}, model: {model_name}): {str(e)}" ) from e - def _check_default_tool_credential(self, tenant_id: str, provider: str, *, session: Session) -> None: + def _check_default_tool_credential(self, tenant_id: str, provider: str, *, credentials: CredentialQuery) -> None: """ Check credential policy compliance for the default workspace credential of a tool provider. @@ -963,29 +369,15 @@ def _check_default_tool_credential(self, tenant_id: str, provider: str, *, sessi :raises ValueError: If no default credential exists or if it fails policy compliance """ try: - from models.tools import BuiltinToolProvider - - # Use the same fallback logic as runtime: get the first available credential - # ordered by is_default DESC, created_at ASC (same as tool_manager.py) - default_provider = session.scalar( - select(BuiltinToolProvider) - .where( - BuiltinToolProvider.tenant_id == tenant_id, - BuiltinToolProvider.provider == provider, - ) - .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc()) - .limit(1) - ) - - if not default_provider: - # plugin does not require credentials, skip + credential_id = credentials.default_tool_credential_id(workspace_id=tenant_id, provider=provider) + if credential_id is None: return # Check credential policy compliance using the default credential ID from core.helper.credential_utils import check_credential_policy_compliance check_credential_policy_compliance( - credential_id=default_provider.id, + credential_id=credential_id, provider=provider, credential_type=PluginCredentialType.TOOL, check_existence=False, @@ -995,7 +387,7 @@ def _check_default_tool_credential(self, tenant_id: str, provider: str, *, sessi raise ValueError(f"Failed to validate default credential for tool provider {provider}: {str(e)}") from e def _validate_load_balancing_credentials( - self, tenant_id: str, node_data: Mapping[str, Any], node_id: str, *, session: Session + self, tenant_id: str, node_data: Mapping[str, Any], node_id: str, *, credentials: CredentialQuery ) -> None: """ Validate load balancing credentials for a workflow node. @@ -1015,17 +407,14 @@ def _validate_load_balancing_credentials( # Check if this model has load balancing enabled if self._is_load_balancing_enabled(tenant_id, provider, model_name): - # Get all load balancing configurations for this model - load_balancing_configs = self._get_load_balancing_configs(tenant_id, provider, model_name, session=session) - # Validate each load balancing configuration + credential_ids = credentials.list_model_load_balancing_credential_ids( + workspace_id=tenant_id, provider=provider, model=model_name + ) try: - for config in load_balancing_configs: - if config.get("credential_id"): - from core.helper.credential_utils import check_credential_policy_compliance + from core.helper.credential_utils import check_credential_policy_compliance - check_credential_policy_compliance( - config["credential_id"], provider, PluginCredentialType.MODEL - ) + for credential_id in credential_ids: + check_credential_policy_compliance(credential_id, provider, PluginCredentialType.MODEL) except Exception as e: raise ValueError(f"Invalid load balancing credentials for {provider}/{model_name}: {str(e)}") from e @@ -1060,47 +449,6 @@ def _is_load_balancing_enabled(self, tenant_id: str, provider: str, model_name: # If we can't determine the status, assume load balancing is not enabled return False - def _get_load_balancing_configs( - self, tenant_id: str, provider: str, model_name: str, *, session: Session - ) -> list[dict[str, Any]]: - """ - Get all load balancing configurations for a model. - - :param tenant_id: The tenant ID - :param provider: The provider name - :param model_name: The model name - :return: List of load balancing configuration dictionaries - """ - try: - from services.model_load_balancing_service import ModelLoadBalancingService - - model_load_balancing_service = ModelLoadBalancingService() - _, configs = model_load_balancing_service.get_load_balancing_configs( - tenant_id=tenant_id, - provider=provider, - model=model_name, - model_type="llm", # Load balancing is primarily used for LLM models - session=session, - config_from="predefined-model", # Check both predefined and custom models - ) - - _, custom_configs = model_load_balancing_service.get_load_balancing_configs( - tenant_id=tenant_id, - provider=provider, - model=model_name, - model_type="llm", - session=session, - config_from="custom-model", - ) - all_configs = cast(list[dict[str, Any]], configs) + cast(list[dict[str, Any]], custom_configs) - - return [config for config in all_configs if config.get("credential_id")] - - except Exception: - # If we can't get the configurations, return empty list - # This will prevent validation errors from breaking the workflow - return [] - def get_default_block_configs(self) -> Sequence[Mapping[str, object]]: """ Get default block configs @@ -1165,73 +513,59 @@ def get_default_block_config( def run_draft_workflow_node( self, app_model: App, - draft_workflow: Workflow, + draft_workflow: WorkflowSnapshot, node_id: str, user_inputs: Mapping[str, Any], account: Account, query: str = "", files: Sequence[File] | None = None, + *, + variables: WorkflowExecutionVariables, ) -> WorkflowNodeExecutionModel: """ Run draft workflow node """ files = files or [] + node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) + node_type = draft_workflow.get_node_type_from_node_config(node_config) node_data = node_config["data"] - is_agent_preview = is_dify_agent_node_data(node_data) - if is_agent_preview and ( - draft_workflow.tenant_id != app_model.tenant_id or draft_workflow.app_id != app_model.id - ): - raise ValueError("Workflow does not belong to the application") - - with Session(bind=db.engine, expire_on_commit=False) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - draft_var_srv.prefill_conversation_variable_default_values(draft_workflow, user_id=account.id) - if is_start_node_type(node_type): - with Session(bind=db.engine) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - conversation_id = draft_var_srv.get_or_create_conversation( - account_id=account.id, - app=app_model, - workflow=draft_workflow, - ) - if node_type == BuiltinNodeTypes.START: - start_data = StartNodeData.model_validate(node_data, from_attributes=True) - user_inputs = _rebuild_file_for_user_inputs_in_start_node( - tenant_id=draft_workflow.tenant_id, start_node_data=start_data, user_inputs=user_inputs - ) - # init variable pool - variable_pool = _setup_variable_pool( - query=query, - files=files or [], - user_id=account.id, - user_inputs=user_inputs, - workflow=draft_workflow, - node_id=node_id, - # NOTE(QuantumGhost): We rely on `DraftVarLoader` to load conversation variables. - conversation_variables=[], - node_type=node_type, - conversation_id=conversation_id, + conversation_id = variables.get_or_create_conversation( + account_id=account.id, + app=app_model, + workflow=draft_workflow, + ) + if node_type == BuiltinNodeTypes.START: + start_data = StartNodeData.model_validate(node_data, from_attributes=True) + user_inputs = _rebuild_file_for_user_inputs_in_start_node( + tenant_id=draft_workflow.tenant_id, start_node_data=start_data, user_inputs=user_inputs ) + # init variable pool + variable_pool = _setup_variable_pool( + query=query, + files=files or [], + user_id=account.id, + user_inputs=user_inputs, + workflow=draft_workflow, + node_id=node_id, + # NOTE(QuantumGhost): We rely on `DraftVarLoader` to load conversation variables. + conversation_variables=[], + node_type=node_type, + conversation_id=conversation_id, + ) else: variable_pool = VariablePool() add_variables_to_pool( variable_pool, build_bootstrap_variables( - system_variables=build_system_variables(), - environment_variables=draft_workflow.environment_variables, + system_variables=default_system_variables(), + environment_variables=environment_variable_values(draft_workflow), ), ) - variable_loader = DraftVarLoader( - engine=db.engine, - app_id=app_model.id, - tenant_id=app_model.tenant_id, - user_id=account.id, - ) + variable_loader = variables.workflow_loader(draft_workflow, account.id) enclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) if enclosing_node_type_and_id: @@ -1239,8 +573,12 @@ def run_draft_workflow_node( else: enclosing_node_id = None + if self._runtime is None: + raise ValueError("Workflow execution dependencies are required") run = WorkflowEntry.single_step_run( - execution_driver=WorkflowRunAgg.run, + workflow_runtime=self._runtime, + agent_binding_resolver=self._runtime.agent_bindings, + draft_variable_saver=variables.saver_factory, workflow=draft_workflow, node_id=node_id, user_inputs=user_inputs, @@ -1249,96 +587,77 @@ def run_draft_workflow_node( variable_loader=variable_loader, ) - repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=db.engine, + repository = self._runtime.node_writer( tenant_id=app_model.tenant_id, user=account, app_id=app_model.id, triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, ) node, node_events = run - if is_agent_preview: - # Standalone previews have no WorkflowRun. The general debug bootstrap - # inserts a synthetic run ID; do not use it as this caller's owner. - variable_pool.remove(system_variable_selector(SystemVariableKey.WORKFLOW_EXECUTION_ID)) - started_event: NodeRunStartedEvent | None = None - started_execution: WorkflowNodeExecution | None = None + workflow_execution_id = get_system_text(variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID) def persist_node_start() -> Generator[NodeEvent | ContainerAwaitRequest]: - nonlocal started_event, started_execution + nonlocal started_event for event in node_events: if isinstance(event, NodeRunStartedEvent) and event.node_id == node_id: started_event = event - # Agent workspace allocation requires this caller row before the node body runs. - started_execution = WorkflowNodeExecution( - id=event.id, - node_execution_id=event.id, - workflow_id=draft_workflow.id, - index=1, - predecessor_node_id=event.predecessor_node_id, - node_id=event.node_id, - node_type=event.node_type, - title=event.node_title, - process_data=event.node_run_result.process_data or None, - metadata=event.node_run_result.metadata, - created_at=event.start_at, + # Agent workspace allocation requires the caller row before + # the node body materializes runtime participants. + repository.save_synchronously( + WorkflowNodeExecution( + id=event.id, + node_execution_id=event.id, + workflow_id=draft_workflow.id, + workflow_execution_id=workflow_execution_id, + index=1, + predecessor_node_id=event.predecessor_node_id, + node_id=event.node_id, + node_type=event.node_type, + title=event.node_title, + process_data=event.node_run_result.process_data or None, + metadata=event.node_run_result.metadata, + created_at=event.start_at, + ) ) - repository.save_synchronously(started_execution) yield event start_at = time.perf_counter() - try: - node_execution = self._handle_single_step_result( - invoke_node_fn=lambda: (node, persist_node_start()), - start_at=start_at, - node_id=node_id, - ) - except BaseException: - if is_agent_preview and started_execution is not None: - started_execution.status = WorkflowNodeExecutionStatus.FAILED - started_execution.error = "Agent preview execution interrupted" - started_execution.elapsed_time = time.perf_counter() - start_at - started_execution.finished_at = naive_utc_now() - try: - repository.save_synchronously(started_execution) - # Keep the terminal append for LogStore and other async repositories. - repository.save(started_execution) - except Exception as persistence_error: - logging.getLogger(__name__).warning( - "Agent preview terminal persistence failed (%s)", type(persistence_error).__name__ - ) - raise + node_execution = self._handle_single_step_result( + invoke_node_fn=lambda: (node, persist_node_start()), + start_at=start_at, + node_id=node_id, + ) node_execution.workflow_id = draft_workflow.id + node_execution.workflow_execution_id = workflow_execution_id if started_event is not None: node_execution.id = started_event.id node_execution.node_execution_id = started_event.id node_execution.created_at = started_event.start_at node_execution.predecessor_node_id = started_event.predecessor_node_id + + # Keep the caller row committed for immediate debug reads, then publish + # the terminal append for asynchronous/log-store repository backends. repository.save_synchronously(node_execution) - # LogStore needs its terminal append in addition to the synchronous SQL caller row. repository.save(node_execution) - workflow_node_execution = self._node_execution_service_repo.get_execution_by_id(node_execution.id) + workflow_node_execution = self._node_execution_service_repo.get_execution_by_id( + node_execution.id, tenant_id=app_model.tenant_id + ) if workflow_node_execution is None: raise ValueError(f"WorkflowNodeExecution with id {node_execution.id} not found after saving") - with sessionmaker(db.engine).begin() as session: - outputs = workflow_node_execution.load_full_outputs(session, storage) + outputs = variables.load_execution_outputs(workflow_node_execution) - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=workflow_node_execution.node_id, - node_type=workflow_node_execution.node_type, - enclosing_node_id=enclosing_node_id, - node_execution_id=node_execution.id, - user=account, - ) - draft_var_saver.save(process_data=node_execution.process_data, outputs=outputs) + draft_var_saver = variables.saver_factory(app_model.tenant_id, account)( + app_id=app_model.id, + node_id=workflow_node_execution.node_id, + node_type=workflow_node_execution.node_type, + enclosing_node_id=enclosing_node_id, + node_execution_id=node_execution.id, + ) + draft_var_saver.save(process_data=node_execution.process_data, outputs=outputs) enqueue_draft_node_execution_trace( execution=workflow_node_execution, @@ -1349,351 +668,6 @@ def persist_node_start() -> Generator[NodeEvent | ContainerAwaitRequest]: return workflow_node_execution - def get_human_input_form_preview( - self, - *, - app_model: App, - account: Account, - node_id: str, - inputs: Mapping[str, Any] | None = None, - session: Session, - ) -> Mapping[str, Any]: - """ - Build a human input form preview for a draft workflow. - - Args: - app_model: Target application model. - account: Current account. - node_id: Human input node ID. - inputs: Values used to fill missing upstream variables referenced in form_content. - """ - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - # inputs: values used to fill missing upstream variables referenced in form_content. - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - - rendered_content = node.render_form_content_before_submission() - resolved_default_values = node.resolve_default_values() - node_data = node.node_data - human_input_required = HumanInputRequired( - form_id=node_id, - form_content=rendered_content, - inputs=node_data.inputs, - actions=node_data.user_actions, - node_id=node_id, - node_title=node.title, - resolved_default_values=resolved_default_values, - ) - return human_input_required.model_dump(mode="json") - - def submit_human_input_form_preview( - self, - *, - app_model: App, - account: Account, - node_id: str, - form_inputs: Mapping[str, Any], - inputs: Mapping[str, Any] | None = None, - action: str, - session: Session, - ) -> Mapping[str, Any]: - """ - Submit a human input form preview for a draft workflow. - - Args: - app_model: Target application model. - account: Current account. - node_id: Human input node ID. - form_inputs: Values the user provides for the form's own fields. - inputs: Values used to fill missing upstream variables referenced in form_content. - action: Selected action ID. - """ - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - # inputs: values used to fill missing upstream variables referenced in form_content. - # form_inputs: values the user provides for the form's own fields. - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - node_data = node.node_data - - human_input_service = HumanInputService(session_factory=sessionmaker(db.engine)) - normalized_form_inputs = human_input_service.validate_and_normalize_submission( - tenant_id=app_model.tenant_id, - form_definition=node_data, - selected_action_id=action, - form_data=form_inputs, - ) - - rendered_content = node.render_form_content_before_submission() - selected_action = next( - (user_action for user_action in node_data.user_actions if user_action.id == action), - None, - ) - outputs: dict[str, Any] = dict(normalized_form_inputs) - outputs["__action_id"] = action - outputs["__action_value"] = selected_action.title if selected_action else "" - outputs["__rendered_content"] = node.render_form_content_with_outputs( - rendered_content, - outputs, - node_data.outputs_field_names(), - node_data.inputs, - ) - - enclosing_node_type_and_id = draft_workflow.get_enclosing_node_type_and_id(node_config) - enclosing_node_id = enclosing_node_type_and_id[1] if enclosing_node_type_and_id else None - with sessionmaker(bind=db.engine).begin() as session: - draft_var_saver = DraftVariableSaver( - session=session, - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=node_id, - node_type=BuiltinNodeTypes.HUMAN_INPUT, - node_execution_id=str(uuid.uuid4()), - user=account, - enclosing_node_id=enclosing_node_id, - ) - draft_var_saver.save(outputs=outputs, process_data={}) - - return outputs - - def test_human_input_delivery( - self, - *, - app_model: App, - account: Account, - node_id: str, - delivery_method_id: str, - inputs: Mapping[str, Any] | None = None, - session: Session, - ) -> None: - draft_workflow = self.get_draft_workflow(app_model=app_model, session=session) - if not draft_workflow: - raise ValueError("Workflow not initialized") - - node_config = draft_workflow.get_node_config_by_id(node_id) - node_type = Workflow.get_node_type_from_node_config(node_config) - if node_type != BuiltinNodeTypes.HUMAN_INPUT: - raise ValueError("Node type must be human-input.") - - node_data = HumanInputNodeData.model_validate( - adapt_human_input_node_data_for_graph(node_config["data"]), - from_attributes=True, - ) - delivery_method = self._resolve_human_input_delivery_method( - node_data=node_data, - delivery_method_id=delivery_method_id, - ) - if delivery_method is None: - raise ValueError("Delivery method not found.") - delivery_method = apply_dify_debug_email_recipient( - delivery_method, - enabled=True, - actor_id=account.id, - ) - - variable_pool = self._build_human_input_variable_pool( - app_model=app_model, - workflow=draft_workflow, - node_config=node_config, - manual_inputs=inputs or {}, - user_id=account.id, - ) - node = self._build_human_input_node_for_debugging( - workflow=draft_workflow, - account=account, - node_config=node_config, - variable_pool=variable_pool, - ) - rendered_content = node.render_form_content_before_submission() - resolved_default_values = node.resolve_default_values() - form_id, recipients = self._create_human_input_delivery_test_form( - app_model=app_model, - node_id=node_id, - node_data=node_data, - delivery_method=delivery_method, - rendered_content=rendered_content, - resolved_default_values=resolved_default_values, - ) - test_service = HumanInputDeliveryTestService() - context = DeliveryTestContext( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - node_id=node_id, - node_title=node_data.title, - rendered_content=rendered_content, - template_vars={"form_id": form_id}, - recipients=recipients, - variable_pool=variable_pool, - ) - try: - test_service.send_test(context=context, method=delivery_method) - except DeliveryTestUnsupportedError as exc: - raise ValueError("Delivery method does not support test send.") from exc - except DeliveryTestError as exc: - raise ValueError(str(exc)) from exc - - @staticmethod - def _resolve_human_input_delivery_method( - *, - node_data: HumanInputNodeData, - delivery_method_id: str, - ) -> DeliveryChannelConfig | None: - for method in parse_human_input_delivery_methods(node_data): - if str(method.id) == delivery_method_id: - return method - return None - - def _create_human_input_delivery_test_form( - self, - *, - app_model: App, - node_id: str, - node_data: HumanInputNodeData, - delivery_method: DeliveryChannelConfig, - rendered_content: str, - resolved_default_values: Mapping[str, Any], - ) -> tuple[str, list[DeliveryTestEmailRecipient]]: - repo = HumanInputFormRepositoryImpl(tenant_id=app_model.tenant_id, app_id=app_model.id) - params = FormCreateParams( - workflow_execution_id=None, - node_id=node_id, - form_config=node_data, - rendered_content=rendered_content, - delivery_methods=[delivery_method], - display_in_ui=False, - resolved_default_values=resolved_default_values, - form_kind=HumanInputFormKind.DELIVERY_TEST, - ) - form_entity = repo.create_form(params) - return form_entity.id, self._load_email_recipients(form_entity.id) - - @staticmethod - def _load_email_recipients(form_id: str) -> list[DeliveryTestEmailRecipient]: - logger = logging.getLogger(__name__) - - with Session(bind=db.engine) as session: - recipients = session.scalars( - select(HumanInputFormRecipient).where(HumanInputFormRecipient.form_id == form_id) - ).all() - recipients_data: list[DeliveryTestEmailRecipient] = [] - for recipient in recipients: - if recipient.recipient_type not in {RecipientType.EMAIL_MEMBER, RecipientType.EMAIL_EXTERNAL}: - continue - if not recipient.access_token: - continue - try: - payload = json.loads(recipient.recipient_payload) - except (json.JSONDecodeError, ValueError): - logger.exception("Failed to parse human input recipient payload for delivery test.") - continue - email = payload.get("email") - if email: - recipients_data.append(DeliveryTestEmailRecipient(email=email, form_token=recipient.access_token)) - return recipients_data - - def _build_human_input_node_for_debugging( - self, - *, - workflow: Workflow, - account: Account, - node_config: NodeConfigDict, - variable_pool: VariablePool, - ) -> _DebugHumanInputNode: - _ = workflow, account - node_data = HumanInputNodeData.model_validate(adapt_human_input_node_data_for_graph(node_config["data"])) - return HumanInputNode( - node_id=node_config["id"], - title=node_data.title, - node_data=node_data, - variable_pool=variable_pool, - ) - - def _build_human_input_variable_pool( - self, - *, - app_model: App, - workflow: Workflow, - node_config: NodeConfigDict, - manual_inputs: Mapping[str, Any], - user_id: str, - ) -> VariablePool: - with Session(bind=db.engine, expire_on_commit=False) as session, session.begin(): - draft_var_srv = WorkflowDraftVariableService(session) - draft_var_srv.prefill_conversation_variable_default_values(workflow, user_id=user_id) - - variable_pool = VariablePool() - add_variables_to_pool( - variable_pool, - build_bootstrap_variables( - system_variables=default_system_variables(), - environment_variables=workflow.environment_variables, - ), - ) - - variable_loader = DraftVarLoader( - engine=db.engine, - app_id=app_model.id, - tenant_id=app_model.tenant_id, - user_id=user_id, - ) - human_input_node_data = HumanInputNodeData.model_validate( - adapt_human_input_node_data_for_graph(node_config["data"]) - ) - variable_mapping = get_human_input_form_variable_mapping(node_config["id"], human_input_node_data) - normalized_user_inputs: dict[str, Any] = dict(manual_inputs) - - load_into_variable_pool( - variable_loader=variable_loader, - variable_pool=variable_pool, - variable_mapping=variable_mapping, - user_inputs=normalized_user_inputs, - ) - WorkflowEntry.mapping_user_inputs_to_variable_pool( - variable_mapping=variable_mapping, - user_inputs=normalized_user_inputs, - variable_pool=variable_pool, - tenant_id=app_model.tenant_id, - ) - - return variable_pool - def run_free_workflow_node( self, node_data: dict[str, Any], tenant_id: str, user_id: str, node_id: str, user_inputs: dict[str, Any] ) -> WorkflowNodeExecution: @@ -1751,6 +725,7 @@ def _handle_single_step_result( created_at=naive_utc_now(), finished_at=naive_utc_now(), ) + # Populate execution result data self._populate_execution_result(node_execution, node_run_result, run_succeeded, error) @@ -1848,43 +823,6 @@ def _populate_execution_result( node_execution.status = WorkflowNodeExecutionStatus.FAILED node_execution.error = error - def convert_to_workflow(self, app_model: App, account: Account, args: dict[str, Any], *, session: Session) -> App: - """ - Basic mode of chatbot app(expert mode) to workflow - Completion App to Workflow App - - :param app_model: App instance - :param account: Account instance - :param args: dict - :return: - """ - # chatbot convert to workflow mode - workflow_converter = WorkflowConverter() - - if app_model.mode not in {AppMode.CHAT, AppMode.COMPLETION}: - raise ValueError(f"Current App mode: {app_model.mode} is not supported convert to workflow.") - - source_app_id = app_model.id - # convert to workflow - new_app: App = workflow_converter.convert_to_workflow( - app_model=app_model, - account=account, - name=args.get("name", "Default Name"), - icon_type=args.get("icon_type", "emoji"), - icon=args.get("icon", "🤖"), - icon_background=args.get("icon_background", "#FFEAD5"), - session=session, - ) - - if SystemFeatureService.is_webapp_auth_enabled(): - # The converter committed and expired the App. Read its identity without - # opening another database transaction before calling Enterprise. - identity = inspect(new_app).identity - assert identity is not None - EnterpriseService.WebAppAuth.inherit_app_access_mode(source_app_id, str(identity[0])) - - return new_app - def validate_graph_structure(self, graph: Mapping[str, Any]): """ Validate workflow graph structure. @@ -1940,132 +878,19 @@ def _validate_human_input_node_data(self, node_data: dict[str, Any]) -> None: Raises: ValueError: If the node data format is invalid """ - from models.human_input_entities import HumanInputNodeData try: HumanInputNodeData.model_validate(adapt_human_input_node_data_for_graph(node_data)) except Exception as e: raise ValueError(f"Invalid HumanInput node data: {str(e)}") from e - def update_workflow( - self, - *, - session: Session, - account_id: str, - data: dict[str, Any], - workflow_ref: WorkflowRef, - ) -> Workflow | None: - """ - Update workflow attributes - - :param session: SQLAlchemy database session - :param account_id: Account ID (for permission check) - :param data: Dictionary containing fields to update - :param workflow_ref: Owner-bound workflow reference - :return: Updated workflow or None if not found - """ - stmt = select(Workflow).where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, - ) - workflow = session.scalar(stmt) - - if not workflow: - return None - - allowed_fields = ["marked_name", "marked_comment"] - - for field, value in data.items(): - if field in allowed_fields: - setattr(workflow, field, value) - - workflow.updated_by = account_id - workflow.updated_at = naive_utc_now() - - return workflow - - def delete_workflow(self, *, session: Session, workflow_ref: WorkflowRef) -> list[str]: - """Stage a published Workflow and its binding owners for deletion. - - The exact owner key is tenant, App, Workflow, and Workflow version. The - Workflow row lock serializes source-version reads and restoration with - deletion. The caller must commit successfully before retiring the - returned, sorted and deduplicated inline Agent candidates. - - :param session: SQLAlchemy database session - :param workflow_ref: Owner-bound workflow reference - :return: Inline Agent IDs whose owner binding is staged for deletion - :raises: ValueError if workflow not found - :raises: WorkflowInUseError if workflow is in use - :raises: DraftWorkflowDeletionError if workflow is a draft version - """ - stmt = ( - select(Workflow) - .where( - Workflow.id == workflow_ref.workflow_id, - Workflow.tenant_id == workflow_ref.tenant_id, - Workflow.app_id == workflow_ref.owner_id, - ) - .with_for_update() - ) - workflow = session.scalar(stmt) - - if not workflow: - raise ValueError(f"Workflow with ID {workflow_ref.workflow_id} not found") - - # Check if workflow is a draft version - if workflow.version == Workflow.VERSION_DRAFT: - raise DraftWorkflowDeletionError("Cannot delete draft workflow versions") - - # Check if this workflow is currently referenced by an app - app_stmt = select(App).where(App.workflow_id == workflow_ref.workflow_id) - app = session.scalar(app_stmt) - if app: - # Cannot delete a workflow that's currently in use by an app - raise WorkflowInUseError(f"Cannot delete workflow that is currently in use by app '{app.id}'") - - # Don't use workflow.tool_published as it's not accurate for specific workflow versions - # Check if there's a tool provider using this specific workflow version - tool_provider = session.scalar( - select(WorkflowToolProvider).where( - WorkflowToolProvider.tenant_id == workflow.tenant_id, - WorkflowToolProvider.app_id == workflow.app_id, - WorkflowToolProvider.version == workflow.version, - ) - ) - - if tool_provider: - # Cannot delete a workflow that's published as a tool - raise WorkflowInUseError("Cannot delete workflow that is published as a tool") - - bindings = session.scalars( - select(WorkflowAgentNodeBinding).where( - WorkflowAgentNodeBinding.tenant_id == workflow.tenant_id, - WorkflowAgentNodeBinding.app_id == workflow.app_id, - WorkflowAgentNodeBinding.workflow_id == workflow.id, - WorkflowAgentNodeBinding.workflow_version == workflow.version, - ) - ).all() - retirement_candidates = sorted( - { - binding.agent_id - for binding in bindings - if binding.binding_type == WorkflowAgentBindingType.INLINE_AGENT and binding.agent_id - } - ) - for binding in bindings: - session.delete(binding) - session.delete(workflow) - return retirement_candidates - def _setup_variable_pool( query: str, files: Sequence[File], user_id: str, user_inputs: Mapping[str, Any], - workflow: Workflow, + workflow: WorkflowSnapshot, node_id: str, node_type: NodeType, conversation_id: str, @@ -2102,7 +927,7 @@ def _setup_variable_pool( variable_pool, build_bootstrap_variables( system_variables=system_variable, - environment_variables=workflow.environment_variables, + environment_variables=environment_variable_values(workflow), conversation_variables=cast(list[Variable], conversation_variables), ), ) diff --git a/api/services/workflow_tool_source_service.py b/api/services/workflow_tool_source_service.py index 1aef10dace38d7..5999e608835a6e 100644 --- a/api/services/workflow_tool_source_service.py +++ b/api/services/workflow_tool_source_service.py @@ -1,18 +1,21 @@ """Materialize pinned Workflow Tool definitions before execution begins.""" -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import Any from pydantic import BaseModel, ConfigDict +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.tool_entities import ToolEntity from core.tools.workflow_as_tool.repository import WorkflowToolSource -from core.tools.workflow_as_tool.tool import WorkflowTool from core.workflow.workflow_tool_container_types import WorkflowToolContainerPayload from graphon.nodes.tool.exc import ToolRuntimeResolutionError +from models import Account from repositories.workflow_tool_source_repository import SQLAlchemyWorkflowToolSourceRepository +from services.tools.workflow.tool import WorkflowTool +from services.workflow.execution.ports import WorkflowRuntime class WorkflowToolDefinition(BaseModel): @@ -55,8 +58,16 @@ def snapshot(self) -> WorkflowToolSnapshot: class WorkflowToolSourceService: - def __init__(self, repository: SQLAlchemyWorkflowToolSourceRepository) -> None: + def __init__( + self, + repository: SQLAlchemyWorkflowToolSourceRepository, + *, + workflow_runtime: WorkflowRuntime | None, + draft_variable_saver: Callable[[str, Account], DraftVariableSaverFactory] | None, + ) -> None: self._repository = repository + self._workflow_runtime = workflow_runtime + self._draft_variable_saver = draft_variable_saver def load( self, @@ -95,6 +106,7 @@ def load( raise ValueError("Workflow Tool source was not found") sources[source.workflow_id] = source pending.append(source.graph_config) + runtime = self._require_runtime() tools[provider_id] = WorkflowTool( workflow_app_id=source.app_id, workflow_id=source.workflow_id, @@ -104,6 +116,9 @@ def load( entity=definition.entity.model_copy(deep=True), runtime=ToolRuntime(tenant_id=tenant_id), label=definition.label, + queries=runtime.tools, + draft_variable_saver=self._draft_variable_saver, + workflow_runtime=runtime, ) visited: set[str] = set() @@ -126,6 +141,8 @@ def load( pending = [] for source in self._repository.get_by_provider_ids(tenant_id=tenant_id, provider_ids=tuple(provider_ids)): if source.tool is not None: + if not isinstance(source.tool, WorkflowTool): + raise TypeError("Workflow Tool source returned an unsupported tool implementation") tools[source.tool.workflow_as_tool_id] = source.tool if source.workflow_id not in sources: pending.append(source.graph_config) @@ -137,3 +154,8 @@ def load( ): raise ToolRuntimeResolutionError("Human Input V2 nodes are not supported in nested Workflow Tools.") return MaterializedWorkflowTools(sources=sources, tools=tools) + + def _require_runtime(self) -> WorkflowRuntime: + if self._workflow_runtime is None: + raise ValueError("Workflow execution dependencies are required to materialize Workflow Tools") + return self._workflow_runtime diff --git a/api/tasks/app_generate/workflow_execute_task.py b/api/tasks/app_generate/workflow_execute_task.py index 12eec51ec5491b..7cf526d4c945b6 100644 --- a/api/tasks/app_generate/workflow_execute_task.py +++ b/api/tasks/app_generate/workflow_execute_task.py @@ -1,7 +1,8 @@ import contextlib import logging import uuid -from collections.abc import Generator, Mapping +from collections.abc import Callable, Generator, Mapping +from dataclasses import replace from enum import StrEnum from functools import partial from typing import Annotated, Any @@ -12,10 +13,7 @@ from sqlalchemy import Engine, select from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator from core.app.apps.execution_coordinator import clear_app_task_cancellation_signals -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import ( AdvancedChatAppGenerateEntity, InvokeFrom, @@ -24,6 +22,7 @@ from core.app.entities.task_entities import WorkflowFinishStreamResponse, WorkflowStartStreamResponse from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext from core.repositories import DifyCoreRepositoryFactory +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from graphon.engine.filter import ResponseStreamFilter from graphon.entities import WorkflowStartReason @@ -38,7 +37,19 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_api_workflow_run_repository import WorkflowRunNotPausedError +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import workflow_snapshot +from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository +from services.agent.retirement_service import WorkflowAgentRetirementService from services.app_task_service import AppTaskService +from services.errors.workflow_service import WorkflowDebugReservationExpiredError +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.debug_cancellation import DebugExecutionCancellation +from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator +from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter +from services.workflow.execution.adapters.response_stream import WorkflowEventStream +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_execution_identity import build_workflow_execution_identity from services.workflow_run_agg import WorkflowRunAgg @@ -99,13 +110,15 @@ class AppExecutionParams(BaseModel): streaming: bool = True call_depth: int = 0 root_node_id: str | None = None + # Only trigger debugging pins a consumed event to a detached draft revision. + workflow_snapshot: WorkflowSnapshot | None = None workflow_run_id: str = Field(default_factory=lambda: str(uuid.uuid4())) @classmethod def new( cls, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, args: Mapping[str, Any], invoke_from: InvokeFrom, @@ -113,6 +126,7 @@ def new( call_depth: int = 0, root_node_id: str | None = None, workflow_run_id: str | None = None, + workflow_snapshot: WorkflowSnapshot | None = None, ): user_params: _Account | _EndUser match user: @@ -133,16 +147,31 @@ def new( streaming=streaming, call_depth=call_depth, root_node_id=root_node_id, + workflow_snapshot=workflow_snapshot, workflow_run_id=workflow_run_id or str(uuid.uuid4()), ) class _AppRunner: - def __init__(self, session_factory: sessionmaker | Engine, exec_params: AppExecutionParams): + def __init__( + self, + session_factory: sessionmaker | Engine, + exec_params: AppExecutionParams, + *, + variables: WorkflowExecutionVariables, + ): if isinstance(session_factory, Engine): session_factory = sessionmaker(bind=session_factory) self._session_factory = session_factory self._exec_params = exec_params + self._variables = variables + self._runtime = build_workflow_execution_dependencies(session_factory) + self._validated_snapshot = False + self._execution_finalized = False + self._cancellation: DebugExecutionCancellation | None = None + + def _mark_execution_finalized(self) -> None: + self._execution_finalized = True @contextlib.contextmanager def _session(self): @@ -157,66 +186,167 @@ def _setup_flask_context(self, user: Account | EndUser): yield def run(self): + try: + return self._run() + except WorkflowDebugReservationExpiredError as exc: + # claim() runs only after validating the persisted owner and snapshot. + # An expired request needs to close SSE; duplicate delivery must not + # emit a failure into the stream owned by an already running worker. + if self._exec_params.streaming and not self._execution_finalized: + _publish_failed_workflow_terminal_events(exc=exc, exec_params=self._exec_params) + raise + finally: + snapshot = self._exec_params.workflow_snapshot + if ( + not self._execution_finalized + and self._validated_snapshot + and snapshot is not None + and snapshot.execution_id is not None + ): + WorkflowAgentRetirementService.finish_execution( + sessions=self._session_factory, + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=snapshot.created_by, + ) + + def _run(self): exec_params = self._exec_params + completed_run: WorkflowRun | None = None + completed_creator: Account | EndUser | None = None with self._session() as session: - workflow = session.get(Workflow, exec_params.workflow_id) - if workflow is None: + snapshot = exec_params.workflow_snapshot + reserved_run = None + if snapshot is not None and snapshot.execution_id is not None: + reserved_run = session.scalar( + select(WorkflowRun) + .where( + WorkflowRun.id == exec_params.workflow_run_id, + WorkflowRun.tenant_id == exec_params.tenant_id, + WorkflowRun.app_id == exec_params.app_id, + WorkflowRun.workflow_id == exec_params.workflow_id, + ) + .with_for_update() + ) + if reserved_run is None or ( + snapshot.execution_id, + snapshot.tenant_id, + snapshot.app_id, + snapshot.id, + ) != (reserved_run.id, reserved_run.tenant_id, reserved_run.app_id, reserved_run.workflow_id): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") + persisted_workflow = session.get(Workflow, exec_params.workflow_id) + if persisted_workflow is None: logger.warning("Workflow %s not found for execution", exec_params.workflow_id) return None - app = session.get(App, workflow.app_id) + app = session.get(App, persisted_workflow.app_id) if app is None: - logger.warning("App %s not found for workflow %s", workflow.app_id, exec_params.workflow_id) + logger.warning("App %s not found for workflow %s", persisted_workflow.app_id, exec_params.workflow_id) return None - pause_config = PauseStateLayerConfig( - session_factory=self._session_factory, - state_owner_user_id=workflow.created_by, - ) + snapshot = exec_params.workflow_snapshot + if snapshot is not None: + if ( + (snapshot.execution_id is not None and snapshot.execution_id != exec_params.workflow_run_id) + or exec_params.invoke_from != InvokeFrom.DEBUGGER + or exec_params.app_mode != AppMode.WORKFLOW + or app.mode != AppMode.WORKFLOW + or (persisted_workflow.id, persisted_workflow.tenant_id, persisted_workflow.app_id) + != (exec_params.workflow_id, exec_params.tenant_id, exec_params.app_id) + or (snapshot.id, snapshot.tenant_id, snapshot.app_id) + != (persisted_workflow.id, app.tenant_id, app.id) + ): + raise ValueError("Workflow snapshot does not belong to this debugger invocation") + workflow = snapshot + if reserved_run is not None: + workflow = replace(snapshot, graph=reserved_run.graph or snapshot.graph) + if WorkflowDebugReservationRepository.claim(session, reserved_run, naive_utc_now()): + self._validated_snapshot = True + else: + completed_run = reserved_run + self._mark_execution_finalized() + if exec_params.streaming: + completed_creator = _resolve_user_for_run(session, reserved_run) + else: + workflow = workflow_snapshot(persisted_workflow) + + if completed_run is not None: + # Redelivery never claims a lease, runs the engine, or repeats cleanup. + # Replay the stored outcome after releasing the read transaction, + # including failures recorded by expiry recovery before worker startup. + if completed_creator is not None and completed_run.finished_at is not None: + response = WorkflowResponseConverter.workflow_run_result_to_finish_response( + task_id=exec_params.workflow_run_id, + workflow_run=completed_run, + creator_user=completed_creator, + ) + topic = WorkflowEventStream.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) + topic.publish(json.dumps(response.model_dump(mode="json"), ensure_ascii=False).encode()) + return None + + with contextlib.nullcontext() if self._validated_snapshot else contextlib.nullcontext() as self._cancellation: + pause_config = PauseStateLayerConfig( + session_factory=self._session_factory, + state_owner_user_id=workflow.created_by, + ) - user = self._resolve_user() + user = self._resolve_user() - with self._setup_flask_context(user), self._session_factory(expire_on_commit=False) as session: - try: - response = self._run_app( - app=app, - workflow=workflow, - user=user, - pause_state_config=pause_config, - session=session, - ) - except Exception as exc: - if exec_params.streaming: - _publish_failed_workflow_terminal_events( - exc=exc, - exec_params=exec_params, + with self._setup_flask_context(user): + try: + response = self._run_app( + app=app, + workflow=workflow, + user=user, + pause_state_config=pause_config, ) - raise - - if not exec_params.streaming: - return response - - assert isinstance(response, Generator) - _publish_streaming_response( - response, - exec_params.workflow_run_id, - exec_params.app_mode, - exec_params.workflow_id, - exec_params.args.get("inputs", {}), - WorkflowStartReason.INITIAL, - ) + except Exception as exc: + if exec_params.streaming: + _publish_failed_workflow_terminal_events( + exc=exc, + exec_params=exec_params, + ) + raise + + if not exec_params.streaming: + return response + + assert isinstance(response, Generator) + _publish_streaming_response( + response, + exec_params.workflow_run_id, + exec_params.app_mode, + exec_params.workflow_id, + exec_params.args.get("inputs", {}), + WorkflowStartReason.INITIAL, + sessions=self._session_factory, + tenant_id=app.tenant_id, + app_id=app.id, + agent_snapshot=exec_params.workflow_snapshot is not None + and exec_params.workflow_snapshot.execution_id is not None, + on_finalized=self._mark_execution_finalized, + on_terminal=self._cancellation.finish if self._cancellation is not None else None, + ) def _run_app( self, *, app: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, pause_state_config: PauseStateLayerConfig, - session: Session, ): + variables = self._variables exec_params = self._exec_params if exec_params.app_mode == AppMode.ADVANCED_CHAT: - return AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run).generate( + return AdvancedChatAppGenerator( + runtime=self._runtime, + execution_driver=WorkflowRunAgg.run, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ).generate( app_model=app, workflow=workflow, user=user, @@ -225,10 +355,15 @@ def _run_app( streaming=exec_params.streaming, workflow_run_id=exec_params.workflow_run_id, pause_state_config=pause_state_config, - session=session, ) if exec_params.app_mode == AppMode.WORKFLOW: - return WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run).generate( + return WorkflowAppGenerator( + runtime=self._runtime, + execution_driver=WorkflowRunAgg.run, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + cancellation=self._cancellation, + ).generate( app_model=app, workflow=workflow, user=user, @@ -282,7 +417,7 @@ def _publish_failed_workflow_terminal_events(exc: Exception, exec_params: AppExe timestamp = to_timestamp(naive_utc_now()) assert timestamp is not None - topic = MessageBasedAppGenerator.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) + topic = WorkflowEventStream.get_response_topic(exec_params.app_mode, exec_params.workflow_run_id) started_payload = WorkflowStartStreamResponse( task_id=exec_params.workflow_run_id, workflow_run_id=exec_params.workflow_run_id, @@ -364,13 +499,24 @@ def _publish_streaming_response( workflow_id: str, inputs: Mapping[str, Any], started_reason: WorkflowStartReason, + *, + sessions: sessionmaker[Session], + tenant_id: str, + app_id: str, + agent_snapshot: bool, + on_finalized: Callable[[], None] | None = None, + on_terminal: Callable[[], None] | None = None, ) -> None: - """Publish workflow stream events and close broken streams with a failed terminal event. + """Publish engine outcomes, using synthetic failures only before an outcome is known. - `_AppRunner.run()` only handles failures before the generator is returned. - Once we start iterating the runtime stream, this helper becomes the last - place that can guarantee SSE consumers eventually see a terminal workflow - lifecycle event. + `_AppRunner.run()` provides fallback execution cleanup until this helper + commits the execution state. Resource cleanup runs after terminal delivery + and remains independently retryable. An observed pause or completion remains authoritative even when + delivery fails; retry its actual event once without inventing a failed result. + + Notify the initial runner after execution finalization succeeds, even when + streaming raises. Its fallback must not infer a second outcome from database + state that asynchronous persistence may have changed in the meantime. """ normalized_workflow_run_id = str(workflow_run_id) @@ -420,74 +566,137 @@ def _publish_failed_terminal_event(error_message: str, task_id: str, publish_sta terminal_events = {"workflow_finished", "workflow_paused"} unexpected_stream_end_message = "Workflow stream ended without a terminal event" - topic = MessageBasedAppGenerator.get_response_topic(app_mode, normalized_workflow_run_id) + topic = WorkflowEventStream.get_response_topic(app_mode, normalized_workflow_run_id) started_published = False terminal_published = False last_task_id = normalized_workflow_run_id stream_error_message: str | None = None + terminal_status: WorkflowExecutionStatus | None = None + terminal_payload: str | None = None + finalized = False + + def finalize() -> None: + nonlocal finalized + if agent_snapshot and not finalized: + WorkflowExecutionWriteRepository(sessions).finish( + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=normalized_workflow_run_id, + status=terminal_status, + ) + finalized = True + if on_finalized is not None: + on_finalized() + try: - for event in response_stream: - event_name = _get_event_name(event) - task_id = _get_task_id(event) - if task_id is not None: - last_task_id = task_id + try: + for event in response_stream: + event_name = _get_event_name(event) + task_id = _get_task_id(event) + if task_id is not None: + last_task_id = task_id + + # Engine outcomes own execution state. Serialization and Redis + # delivery can fail while a paused execution still needs its Agents. + if event_name == "workflow_paused": + terminal_status = WorkflowExecutionStatus.PAUSED + elif event_name == "workflow_finished": + data = _get_event_data(event) or {} + terminal_status = WorkflowExecutionStatus(data.get("data", {}).get("status", "failed")) + + if event_name in terminal_events: + if on_terminal is not None: + on_terminal() + # A subscriber can resume immediately on receiving PAUSED. + # Commit and acknowledge this worker's outcome first; neither + # this helper nor the outer runner may finalize it again. + finalize() + + try: + if isinstance(event, BaseModel): + payload = json.dumps(event.model_dump(mode="json"), ensure_ascii=False) + else: + payload = json.dumps(event, ensure_ascii=False, default=str) + except (TypeError, ValueError): + logger.exception("error while encoding event") + continue + + if event_name in terminal_events: + terminal_payload = payload + topic.publish(payload.encode()) + + if event_name == "workflow_started": + started_published = True + elif event_name in terminal_events: + terminal_published = True + elif event_name == "error": + stream_error_message = _get_error_message(event) or stream_error_message + except Exception as exc: + if not terminal_published and terminal_payload is not None: + logger.exception("Retrying engine terminal event publication for run %s", normalized_workflow_run_id) + topic.publish(terminal_payload.encode()) + elif not terminal_published and terminal_status is None: + logger.exception( + "Workflow stream for run %s failed before terminal event; publishing fallback terminal event", + normalized_workflow_run_id, + ) + _publish_failed_terminal_event( + error_message=str(exc) or exc.__class__.__name__, + task_id=last_task_id, + publish_started=not started_published, + ) + raise - try: - if isinstance(event, BaseModel): - payload = json.dumps(event.model_dump(mode="json"), ensure_ascii=False) - else: - payload = json.dumps(event, ensure_ascii=False, default=str) - except (TypeError, ValueError): - logger.exception("error while encoding event") - continue - - topic.publish(payload.encode()) - - if event_name == "workflow_started": - started_published = True - elif event_name in terminal_events: - terminal_published = True - elif event_name == "error": - stream_error_message = _get_error_message(event) or stream_error_message - except Exception as exc: - if not terminal_published: - logger.exception( - "Workflow stream for run %s failed before terminal event; publishing fallback terminal event", + if not terminal_published and terminal_status is None: + logger.warning( + "Workflow stream for run %s ended without a terminal event; publishing fallback terminal event", normalized_workflow_run_id, ) _publish_failed_terminal_event( - error_message=str(exc) or exc.__class__.__name__, + error_message=stream_error_message or unexpected_stream_end_message, task_id=last_task_id, publish_started=not started_published, ) - raise - - if not terminal_published: - logger.warning( - "Workflow stream for run %s ended without a terminal event; publishing fallback terminal event", - normalized_workflow_run_id, - ) - _publish_failed_terminal_event( - error_message=stream_error_message or unexpected_stream_end_message, - task_id=last_task_id, - publish_started=not started_published, - ) + finally: + try: + if isinstance(response_stream, Generator): + response_stream.close() + finally: + finalize() + if agent_snapshot and finalized: + # Cleanup dispatch is independent of the engine outcome and SSE. + # Retain durable retry evidence if the resource broker is down. + try: + WorkflowAgentRetirementService.retire_finished_execution( + delete_unstarted=False, + sessions=sessions, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=normalized_workflow_run_id, + account_id=None, + ) + except Exception: + logger.exception("Failed to dispatch Agent cleanup for run %s", normalized_workflow_run_id) @shared_task(queue=WORKFLOW_BASED_APP_EXECUTION_QUEUE) def workflow_based_app_execution_task( payload: str, ) -> Mapping[str, Any] | None: + from extensions.ext_application_services import application_services + exec_params = AppExecutionParams.model_validate_json(payload) logger.info("workflow_based_app_execution_task run with params: %s", exec_params) - runner = _AppRunner(db.engine, exec_params=exec_params) + runner = _AppRunner(db.engine, exec_params=exec_params, variables=application_services().workflow_variables) return runner.run() -def _resume_app_execution(payload: dict[str, Any]) -> None: +def _resume_app_execution(payload: dict[str, Any], *, variables: WorkflowExecutionVariables) -> None: workflow_run_id = payload["workflow_run_id"] session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) @@ -517,8 +726,8 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: logger.warning("Workflow run %s not found during resume", workflow_run_id) return - workflow = session.get(Workflow, workflow_run.workflow_id) - if workflow is None: + workflow_model = session.get(Workflow, workflow_run.workflow_id) + if workflow_model is None: logger.warning("Workflow %s not found during resume", workflow_run.workflow_id) return @@ -557,6 +766,13 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: logger.warning("Message not found for workflow run %s", workflow_run_id) return + workflow = workflow_snapshot(workflow_model) + + binding_snapshot = workflow_run.graph_dict.get("_agent_bindings", {}) + if binding_snapshot.get("execution_id") == workflow_run.id and workflow_run.graph is not None: + # The resumed graph and its Agent generations belong to the original execution. + workflow = replace(workflow, graph=workflow_run.graph) + if not isinstance(generate_entity, (AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity)): logger.error( "Unsupported resumption entity for workflow run %s (found %s)", @@ -611,24 +827,24 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: case AdvancedChatAppGenerateEntity(): assert conversation is not None assert message is not None - with session_factory() as session: - _resume_advanced_chat( - app_model=app_model, - workflow=workflow, - user=user, - conversation=conversation, - message=message, - generate_entity=generate_entity, - graph_runtime_state=graph_runtime_state, - response_stream_filter=response_stream_filter, - session_factory=session_factory, - pause_state_config=pause_config, - workflow_run_id=workflow_run_id, - workflow_run=workflow_run, - session=session, - ) + _resume_advanced_chat( + variables=variables, + app_model=app_model, + workflow=workflow, + user=user, + conversation=conversation, + message=message, + generate_entity=generate_entity, + graph_runtime_state=graph_runtime_state, + response_stream_filter=response_stream_filter, + session_factory=session_factory, + pause_state_config=pause_config, + workflow_run_id=workflow_run_id, + workflow_run=workflow_run, + ) case WorkflowAppGenerateEntity(): _resume_workflow( + variables=variables, app_model=app_model, workflow=workflow, user=user, @@ -646,8 +862,9 @@ def _resume_app_execution(payload: dict[str, Any]) -> None: def _resume_advanced_chat( *, + variables: WorkflowExecutionVariables, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, conversation: Conversation, message: Message, @@ -658,7 +875,6 @@ def _resume_advanced_chat( pause_state_config: PauseStateLayerConfig, workflow_run_id: str, workflow_run: WorkflowRun, - session: Session, ) -> None: resumed_generate_entity = generate_entity.model_copy(update={"stream": True}) @@ -682,7 +898,12 @@ def _resume_advanced_chat( triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) - generator = AdvancedChatAppGenerator(execution_driver=WorkflowRunAgg.run) + generator = AdvancedChatAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + execution_driver=WorkflowRunAgg.run, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + ) try: response = generator.resume( @@ -697,7 +918,6 @@ def _resume_advanced_chat( graph_runtime_state=graph_runtime_state, pause_state_config=pause_state_config, response_stream_filter=response_stream_filter, - session=session, ) except Exception: logger.exception("Failed to resume chatflow execution for workflow run %s", workflow_run_id) @@ -711,13 +931,18 @@ def _resume_advanced_chat( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot="_agent_bindings" in workflow_run.graph_dict, ) def _resume_workflow( *, + variables: WorkflowExecutionVariables, app_model: App, - workflow: Workflow, + workflow: WorkflowSnapshot, user: Account | EndUser, generate_entity: WorkflowAppGenerateEntity, graph_runtime_state: RuntimeState, @@ -751,33 +976,48 @@ def _resume_workflow( triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN, ) - generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run) - - try: - response = generator.resume( - app_model=app_model, - workflow=workflow, - user=user, - application_generate_entity=resumed_generate_entity, - graph_runtime_state=graph_runtime_state, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - pause_state_config=pause_state_config, - response_stream_filter=response_stream_filter, + with ( + contextlib.nullcontext() + if workflow_run.graph_dict.get("_agent_bindings", {}).get("execution_id") == workflow_run_id + else contextlib.nullcontext() + ) as cancellation: + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + execution_driver=WorkflowRunAgg.run, + draft_variable_loader=variables.workflow_loader, + draft_variable_saver=variables.saver_factory, + cancellation=cancellation, ) - except Exception: - logger.exception("Failed to resume workflow execution for workflow run %s", workflow_run_id) - raise + try: + response = generator.resume( + app_model=app_model, + workflow=workflow, + user=user, + application_generate_entity=resumed_generate_entity, + graph_runtime_state=graph_runtime_state, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + pause_state_config=pause_state_config, + response_stream_filter=response_stream_filter, + ) + except Exception: + logger.exception("Failed to resume workflow execution for workflow run %s", workflow_run_id) + raise - assert isinstance(response, Generator) - _publish_streaming_response( - response, - workflow_run_id, - AppMode.WORKFLOW, - workflow.id, - generate_entity.inputs, - WorkflowStartReason.RESUMPTION, - ) + assert isinstance(response, Generator) + _publish_streaming_response( + response, + workflow_run_id, + AppMode.WORKFLOW, + workflow.id, + generate_entity.inputs, + WorkflowStartReason.RESUMPTION, + sessions=session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot="_agent_bindings" in workflow_run.graph_dict, + on_terminal=cancellation.finish if cancellation is not None else None, + ) try: workflow_run_repo.delete_workflow_pause(pause_entity) @@ -792,4 +1032,6 @@ def _resume_workflow( @shared_task(queue=WORKFLOW_BASED_APP_EXECUTION_QUEUE, name="resume_app_execution") def resume_app_execution(payload: dict[str, Any]) -> None: - _resume_app_execution(payload) + from extensions.ext_application_services import application_services + + _resume_app_execution(payload, variables=application_services().workflow_variables) diff --git a/api/tasks/async_workflow_tasks.py b/api/tasks/async_workflow_tasks.py index 4620f3e55bd044..a9d256a402de1a 100644 --- a/api/tasks/async_workflow_tasks.py +++ b/api/tasks/async_workflow_tasks.py @@ -16,13 +16,13 @@ from typing_extensions import TypedDict from configs import dify_config -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerateEntity from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext from core.app.layers.timeslice_layer import TimeSliceLayer from core.app.layers.trigger_post_layer import TriggerPostLayer from core.db.session_factory import session_factory from core.repositories import DifyCoreRepositoryFactory +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from graphon.runtime import RuntimeState from models.account import Account @@ -32,6 +32,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_workflow_trigger_log_repository import SQLAlchemyWorkflowTriggerLogRepository +from repositories.workflow.definition_repository import workflow_snapshot from services.app_task_service import AppTaskService from services.errors.app import WorkflowNotFoundError from services.workflow.entities import ( @@ -39,6 +40,7 @@ WorkflowResumeTaskData, WorkflowTaskData, ) +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator from services.workflow_execution_identity import build_workflow_execution_identity from services.workflow_run_agg import WorkflowRunAgg from tasks.workflow_cfs_scheduler.cfs_scheduler import AsyncWorkflowCFSPlanEntity, AsyncWorkflowCFSPlanScheduler @@ -142,14 +144,18 @@ def _execute_workflow_common( if not app_model: raise WorkflowNotFoundError(f"App not found: {trigger_log.app_id}") - workflow = session.scalar(select(Workflow).where(Workflow.id == trigger_log.workflow_id)) - if not workflow: + workflow_model = session.scalar(select(Workflow).where(Workflow.id == trigger_log.workflow_id)) + if not workflow_model: raise WorkflowNotFoundError(f"Workflow not found: {trigger_log.workflow_id}") + workflow = workflow_snapshot(workflow_model) user = _get_user(session, trigger_log) # Execute workflow using WorkflowAppGenerator - generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run) + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory.get_session_maker()), + execution_driver=WorkflowRunAgg.run, + ) # Adapt trigger inputs and files for the generator. args = _build_generator_args(trigger_data) @@ -256,11 +262,12 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None: response_stream_filter = resumption_context.get_response_stream_filter() with session_factory() as session: - workflow = session.scalar(select(Workflow).where(Workflow.id == workflow_run.workflow_id)) - if workflow is None: + workflow_model = session.scalar(select(Workflow).where(Workflow.id == workflow_run.workflow_id)) + if workflow_model is None: raise WorkflowNotFoundError( f"Workflow not found: workflow_run_id={workflow_run.id}, workflow_id={workflow_run.workflow_id}" ) + workflow = workflow_snapshot(workflow_model) user = _get_user(session, workflow_run) app_model = session.scalar(select(App).where(App.id == workflow_run.app_id)) if app_model is None: @@ -286,7 +293,10 @@ def resume_workflow_execution(task_data_dict: dict[str, Any]) -> None: state_owner_user_id=workflow.created_by, ) - generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run) + generator = WorkflowAppGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + execution_driver=WorkflowRunAgg.run, + ) start_time = datetime.now(UTC) graph_engine_layers = [] trigger_log = _query_trigger_log_info(session_factory, task_data.workflow_run_id) diff --git a/api/tasks/human_input_timeout_tasks.py b/api/tasks/human_input_timeout_tasks.py index 9f499d724004d7..ad2bd91376060a 100644 --- a/api/tasks/human_input_timeout_tasks.py +++ b/api/tasks/human_input_timeout_tasks.py @@ -9,15 +9,18 @@ from configs import dify_config from core.db.session_factory import session_factory as v2_session_factory from core.human_input_v2.shared.values import TenantId -from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from enums.human_input import HumanInputFormKind, HumanInputFormStatus from extensions.ext_database import db from extensions.ext_storage import storage from graphon.enums import WorkflowExecutionStatus from libs.datetime_utils import ensure_naive_utc, naive_utc_now +from models.enums import CreatorUserRole from models.human_input import HumanInputForm from models.workflow import WorkflowPause, WorkflowRun +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from repositories.human_input_v2.sqlalchemy_form_repository import SQLAlchemyFormExpirationRepository +from repositories.workflow.execution_write_repository import sync_debug_lease_status +from services.agent.retirement_service import WorkflowAgentRetirementService from services.human_input_service import HumanInputService from services.human_input_v2.form_service import handle_form_timeouts from tasks.app_generate.workflow_execute_task import WORKFLOW_BASED_APP_EXECUTION_QUEUE @@ -77,25 +80,48 @@ def _is_global_timeout(form_model: HumanInputForm, global_timeout_seconds: int, def _handle_global_timeout(*, form_id: str, workflow_run_id: str, node_id: str, session_factory: sessionmaker) -> None: now = naive_utc_now() with session_factory() as session, session.begin(): - workflow_run = session.get(WorkflowRun, workflow_run_id) - if workflow_run is not None: + workflow_run = session.scalar( + select(WorkflowRun) + .join( + HumanInputForm, + (HumanInputForm.workflow_run_id == WorkflowRun.id) + & (HumanInputForm.tenant_id == WorkflowRun.tenant_id) + & (HumanInputForm.app_id == WorkflowRun.app_id), + ) + .where(HumanInputForm.id == form_id, WorkflowRun.id == workflow_run_id) + .with_for_update(of=WorkflowRun) + ) + if workflow_run is None: + return + if not workflow_run.status.is_ended(): workflow_run.status = WorkflowExecutionStatus.STOPPED workflow_run.error = f"Human input global timeout at node {node_id}" workflow_run.finished_at = now session.add(workflow_run) + sync_debug_lease_status(session, workflow_run) pause_model = session.scalar(select(WorkflowPause).where(WorkflowPause.workflow_run_id == workflow_run_id)) + state_object_key = None if pause_model is not None: - try: - storage.delete(pause_model.state_object_key) - except Exception: - logger.exception( - "Failed to delete pause state object for workflow_run_id=%s, pause_id=%s", - workflow_run_id, - pause_model.id, - ) + state_object_key = pause_model.state_object_key pause_model.resumed_at = now session.add(pause_model) + tenant_id, app_id, workflow_id = workflow_run.tenant_id, workflow_run.app_id, workflow_run.workflow_id + account_id = workflow_run.created_by if workflow_run.created_by_role == CreatorUserRole.ACCOUNT else None + + if state_object_key is not None: + try: + storage.delete(state_object_key) + except Exception: + logger.exception("Failed to delete pause state object for workflow_run_id=%s", workflow_run_id) + WorkflowAgentRetirementService.finish_execution( + sessions=session_factory, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=workflow_run_id, + account_id=account_id, + ) def _handle_legacy_human_input_timeouts(limit: int) -> None: diff --git a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py index 43861940769d66..b5cc010edd796b 100644 --- a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py +++ b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py @@ -14,13 +14,13 @@ from sqlalchemy.orm import Session, sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity from core.db.session_factory import get_session_maker from core.rag.pipeline.queue import TenantIsolatedTaskQueue from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.data_sources import build_data_source_credentials +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from models import Account, Tenant from models.dataset import Pipeline @@ -28,6 +28,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.file_service import FileService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from services.workflow_run_agg import WorkflowRunAgg logger = logging.getLogger(__name__) @@ -180,26 +181,25 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any], # Since we're already in a thread pool, no need for nested threading pipeline_generator = PipelineGenerator( + runtime=build_workflow_execution_dependencies(session_factory), execution_driver=WorkflowRunAgg.run, documents=SQLAlchemyDocumentRepository(session_factory=session_factory), datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, ) # Using protected method intentionally for async execution - with Session(db.engine, expire_on_commit=False) as session: - pipeline_generator._generate( # type: ignore[attr-defined] - session=session, - flask_app=flask_app, - context=context, - pipeline=pipeline, - workflow_id=workflow_id, - user=account, - application_generate_entity=entity, - invoke_from=InvokeFrom.PUBLISHED_PIPELINE, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - streaming=streaming, - workflow_thread_pool_id=workflow_thread_pool_id, - ) + pipeline_generator._generate( # type: ignore[attr-defined] + flask_app=flask_app, + context=context, + pipeline=pipeline, + workflow_id=workflow_id, + user=account, + application_generate_entity=entity, + invoke_from=InvokeFrom.PUBLISHED_PIPELINE, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + streaming=streaming, + workflow_thread_pool_id=workflow_thread_pool_id, + ) except Exception: logging.exception("Error in priority pipeline task") raise diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.py index 26f2a0247d8c0d..bbc0b46533ef9a 100644 --- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py +++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py @@ -15,13 +15,13 @@ from sqlalchemy.orm import sessionmaker from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity from core.db.session_factory import get_session_maker from core.rag.pipeline.queue import TenantIsolatedTaskQueue from core.repositories.factory import DifyCoreRepositoryFactory from extensions.application_services.data_sources import build_data_source_credentials +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_database import db from models import Account, Tenant from models.dataset import Pipeline @@ -29,6 +29,7 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.file_service import FileService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from services.workflow_run_agg import WorkflowRunAgg logger = logging.getLogger(__name__) @@ -158,61 +159,59 @@ def run_single_rag_pipeline_task(rag_pipeline_invoke_entity: Mapping[str, Any], if not workflow: raise ValueError(f"Workflow {pipeline.workflow_id} not found") - if workflow_execution_id is None: - workflow_execution_id = str(uuid.uuid4()) - - # Create application generate entity from dict - entity = RagPipelineGenerateEntity.model_validate(application_generate_entity) - - # Create workflow repositories - session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) - workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( - session_factory=session_factory, - tenant_id=pipeline.tenant_id, - user=account, - app_id=entity.app_config.app_id, - triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN, - ) - - workflow_node_execution_repository = ( - DifyCoreRepositoryFactory.create_workflow_node_execution_repository( - session_factory=session_factory, - tenant_id=pipeline.tenant_id, - user=account, - app_id=entity.app_config.app_id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN, - ) - ) - - # Set the user directly in g for preserve_flask_contexts - g._login_user = account - - # Copy context for passing to pipeline generator - context = contextvars.copy_context() - - # Direct execution without creating another thread - # Since we're already in a thread pool, no need for nested threading - - pipeline_generator = PipelineGenerator( - execution_driver=WorkflowRunAgg.run, - documents=SQLAlchemyDocumentRepository(session_factory=session_factory), - datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, - ) - # Using protected method intentionally for async execution - pipeline_generator._generate( # type: ignore[attr-defined] - session=session, - flask_app=flask_app, - context=context, - pipeline=pipeline, - workflow_id=workflow_id, - user=account, - application_generate_entity=entity, - invoke_from=InvokeFrom.PUBLISHED_PIPELINE, - workflow_execution_repository=workflow_execution_repository, - workflow_node_execution_repository=workflow_node_execution_repository, - streaming=streaming, - workflow_thread_pool_id=workflow_thread_pool_id, - ) + if workflow_execution_id is None: + workflow_execution_id = str(uuid.uuid4()) + + # Create application generate entity from dict + entity = RagPipelineGenerateEntity.model_validate(application_generate_entity) + + # Create workflow repositories + session_factory = sessionmaker(bind=db.engine, expire_on_commit=False) + workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository( + session_factory=session_factory, + tenant_id=pipeline.tenant_id, + user=account, + app_id=entity.app_config.app_id, + triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_RUN, + ) + + workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository( + session_factory=session_factory, + tenant_id=pipeline.tenant_id, + user=account, + app_id=entity.app_config.app_id, + triggered_from=WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN, + ) + + # Set the user directly in g for preserve_flask_contexts + g._login_user = account + + # Copy context for passing to pipeline generator + context = contextvars.copy_context() + + # Direct execution without creating another thread + # Since we're already in a thread pool, no need for nested threading + + pipeline_generator = PipelineGenerator( + runtime=build_workflow_execution_dependencies(session_factory), + execution_driver=WorkflowRunAgg.run, + documents=SQLAlchemyDocumentRepository(session_factory=session_factory), + datasource_providers=build_data_source_credentials(database_client=get_session_maker()).providers, + ) + # Using protected method intentionally for async execution + pipeline_generator._generate( # type: ignore[attr-defined] + flask_app=flask_app, + context=context, + pipeline=pipeline, + workflow_id=workflow_id, + user=account, + application_generate_entity=entity, + invoke_from=InvokeFrom.PUBLISHED_PIPELINE, + workflow_execution_repository=workflow_execution_repository, + workflow_node_execution_repository=workflow_node_execution_repository, + streaming=streaming, + workflow_thread_pool_id=workflow_thread_pool_id, + ) except Exception: logging.exception("Error in pipeline task") raise diff --git a/api/tasks/retry_document_indexing_task.py b/api/tasks/retry_document_indexing_task.py index 5a12b74ff1aab0..3703da1f06f9d0 100644 --- a/api/tasks/retry_document_indexing_task.py +++ b/api/tasks/retry_document_indexing_task.py @@ -6,22 +6,18 @@ from sqlalchemy import delete, select from configs import dify_config -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator -from core.db.session_factory import get_session_maker, session_factory +from core.db.session_factory import session_factory from core.rag.index_processor.index_processor import IndexProcessorFactory from enums import DeploymentEdition -from extensions.application_services.data_sources import build_data_source_credentials from extensions.ext_redis import redis_client from libs.datetime_utils import naive_utc_now from models import Account, Tenant from models.dataset import Dataset, Document, DocumentSegment from models.enums import IndexingStatus -from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository from services.feature_service import FeatureService from services.knowledge.indexing.adapters.execution import build_document_indexing_service from services.knowledge.resource_scope import DatasetRef from services.rag_pipeline.rag_pipeline import RagPipelineService -from services.workflow_run_agg import WorkflowRunAgg logger = logging.getLogger(__name__) @@ -120,21 +116,15 @@ def retry_document_indexing_task(dataset_id: str, document_ids: list[str], user_ session.commit() if runtime_mode == "rag_pipeline": + from extensions.ext_application_services import application_services + with session_factory.create_session() as rag_session: rag_pipeline_service = RagPipelineService(rag_session) rag_pipeline_service.retry_error_document( dataset, document, user, - generator=PipelineGenerator( - execution_driver=WorkflowRunAgg.run, - documents=SQLAlchemyDocumentRepository( - session_factory=session_factory.get_session_maker() - ), - datasource_providers=build_data_source_credentials( - database_client=get_session_maker() - ).providers, - ), + generator=application_services().knowledge.pipeline_generator, ) else: indexing_service = build_document_indexing_service( diff --git a/api/tasks/workflow_draft_var_tasks.py b/api/tasks/workflow_draft_var_tasks.py index 01aab2ac561826..dd62dba2f9fc40 100644 --- a/api/tasks/workflow_draft_var_tasks.py +++ b/api/tasks/workflow_draft_var_tasks.py @@ -1,30 +1,14 @@ -""" -Celery tasks for asynchronous workflow execution storage operations. - -These tasks provide asynchronous storage capabilities for workflow execution data, -improving performance by offloading storage operations to background workers. -""" +"""Retry reclamation of uploads from deleted variables or failed variable saves.""" import logging from celery import shared_task # type: ignore[import-untyped] from core.db.session_factory import session_factory -from services.workflow_draft_variable_service import DraftVarFileDeletion, WorkflowDraftVariableService logger = logging.getLogger(__name__) -@shared_task(queue="workflow_draft_var", bind=True, max_retries=3, default_retry_delay=60) -def save_workflow_execution_task( - self, - deletions: list[DraftVarFileDeletion], -): - with session_factory.create_session() as session, session.begin(): - srv = WorkflowDraftVariableService(session=session) - srv.delete_workflow_draft_variable_file(deletions=deletions) - - @shared_task( queue="workflow_storage", autoretry_for=(Exception,), diff --git a/api/tests/e2e_tests/containers/conftest.py b/api/tests/e2e_tests/containers/conftest.py index 66a09fca1a53f7..146f784c20a86a 100644 --- a/api/tests/e2e_tests/containers/conftest.py +++ b/api/tests/e2e_tests/containers/conftest.py @@ -19,7 +19,7 @@ from flask import Flask from flask.testing import FlaskClient from sqlalchemy import Engine, text -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from testcontainers.core.container import DockerContainer from testcontainers.core.network import Network from testcontainers.core.wait_strategies import HttpWaitStrategy, LogMessageWaitStrategy @@ -28,6 +28,7 @@ from app_factory import create_app from extensions.ext_database import db +from services.workflow.variable_contracts import WorkflowExecutionVariables # Configure logging for test containers logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -545,3 +546,12 @@ def _fake_request(method: str, url: str, **_kwargs: object) -> httpx.Response: patch("core.helper.ssrf_proxy.head", side_effect=lambda url, **kw: _fake_request("HEAD", url, **kw)), ): yield + + +@pytest.fixture +def workflow_variables(db_session_with_containers: Session) -> WorkflowExecutionVariables: + from extensions.application_services.workflow_variables import build_workflow_variable_service + + return build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) diff --git a/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py b/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py index a66d15593a680b..d6bdb39c72cac9 100644 --- a/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py +++ b/api/tests/e2e_tests/containers/controllers/openapi/test_app_dsl.py @@ -53,17 +53,14 @@ def external_deps() -> Generator[dict[str, object]]: workflow sync, dependency analysis, enterprise hooks) while leaving the DSL service and DB writes real.""" with ( - patch("services.app_dsl_service.WorkflowService") as mock_workflow_service, + patch("extensions.application_services.workflow.build_workflow_draft_service") as mock_drafts, patch("services.app_dsl_service.DependenciesAnalysisService") as mock_dependencies_service, patch("services.app_dsl_service.app_was_created") as mock_app_was_created, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, patch("services.app_service.SystemFeatureService") as mock_feature_service, patch("services.app_service.EnterpriseService") as mock_enterprise_service, ): - mock_workflow_service.get_draft_workflow.return_value = None - mock_workflow_service.get_published_workflow_by_id.return_value = None - mock_workflow_service.return_value.get_draft_workflow.return_value = None - mock_workflow_service.return_value.sync_draft_workflow.return_value = None + mock_drafts.return_value.sync.return_value = None mock_dependencies_service.generate_latest_dependencies.return_value = [] # type: ignore[assignment] mock_dependencies_service.get_leaked_dependencies.return_value = [] # type: ignore[assignment] mock_dependencies_service.generate_dependencies.return_value = [] # type: ignore[assignment] @@ -77,7 +74,7 @@ def external_deps() -> Generator[dict[str, object]]: mock_enterprise_service.WebAppAuth.update_app_access_mode.return_value = None mock_enterprise_service.WebAppAuth.cleanup_webapp.return_value = None - yield {"workflow_service": mock_workflow_service} + yield {} def _app_and_account(db_session: Session, *, mode: str = "chat") -> tuple[App, Account]: diff --git a/api/tests/e2e_tests/containers/core/app/layers/test_pause_state_persist_layer.py b/api/tests/e2e_tests/containers/core/app/layers/test_pause_state_persist_layer.py index d924462f5219fc..7bd79212be2008 100644 --- a/api/tests/e2e_tests/containers/core/app/layers/test_pause_state_persist_layer.py +++ b/api/tests/e2e_tests/containers/core/app/layers/test_pause_state_persist_layer.py @@ -12,14 +12,15 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.app_config.entities import WorkflowUIBasedAppConfig -from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager -from core.app.apps.workflow_app_runner import PreparedWorkflowRun, WorkflowBasedAppRunner from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig, WorkflowResumptionContext from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository from core.workflow.system_variables import build_system_variables -from core.workflow.workflow_entry import WorkflowEntry +from extensions.application_services.workflow import ( + WorkflowExecutionDependencies, + build_workflow_execution_dependencies, +) from extensions.ext_storage import storage from graphon.engine.command import InMemoryChannel, PauseCommand from graphon.engine_events import GraphRunFailedEvent, GraphRunPausedEvent, GraphRunStartedEvent, GraphRunSucceededEvent @@ -27,6 +28,7 @@ from graphon.model_runtime.entities.llm_entities import LLMUsage from graphon.nodes.start.entities import StartNodeData from graphon.runtime import RuntimeState, VariablePool +from graphon.variable_loader import DUMMY_VARIABLE_LOADER from models import Account from models.account import Tenant, TenantAccountJoin, TenantAccountRole from models.enums import WorkflowRunTriggeredFrom @@ -34,6 +36,11 @@ from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom, WorkflowPause, WorkflowRun from models.workflow import WorkflowType as ModelWorkflowType from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from repositories.workflow.definition_repository import workflow_snapshot +from services.workflow.execution.adapters.workflow.app_queue_manager import WorkflowAppQueueManager +from services.workflow.execution.adapters.workflow.app_runner import WorkflowAppRunner +from services.workflow.execution.adapters.workflow_entry import WorkflowEntry +from services.workflow.execution.contracts import PreparedWorkflowRun from services.workflow_run_agg import WorkflowRunAgg @@ -44,6 +51,7 @@ class WorkflowCase: account: Account workflow: Workflow run_id: str + runtime: WorkflowExecutionDependencies def prepare( self, @@ -71,7 +79,14 @@ def prepare( workflow_execution_id=self.run_id, ) queue = WorkflowAppQueueManager(entity.task_id, self.account.id, entity.invoke_from, AppMode.WORKFLOW) - runner = WorkflowBasedAppRunner(queue_manager=queue, app_id=self.workflow.app_id) + runner = WorkflowAppRunner( + runtime=self.runtime, + application_generate_entity=entity, + queue_manager=queue, + variable_loader=DUMMY_VARIABLE_LOADER, + workflow=workflow_snapshot(self.workflow), + system_user_id=self.account.id, + ) variable_pool = VariablePool.from_bootstrap( system_variables=build_system_variables( app_id=self.workflow.app_id, workflow_id=self.workflow.id, workflow_execution_id=self.run_id @@ -89,7 +104,7 @@ def prepare( outputs=dict(outputs or {}), node_run_steps=node_run_steps, ) - graph = runner._init_graph( + graph = runner._graphs.build( graph_config=self.workflow.graph_dict, graph_runtime_state=runtime, workflow_id=self.workflow.id, @@ -117,6 +132,7 @@ def prepare( if pause: commands.send_command(PauseCommand(reason="integration pause")) entry = WorkflowEntry( + human_form_reader=runner.human_form_reader, graph=graph, graph_config=self.workflow.graph_dict, call_depth=0, @@ -168,7 +184,14 @@ def workflow_case(db_session_with_containers: Session) -> WorkflowCase: ) session.add(workflow) session.commit() - return WorkflowCase(session, engine, account, workflow, str(uuid4())) + return WorkflowCase( + session, + engine, + account, + workflow, + str(uuid4()), + build_workflow_execution_dependencies(sessionmaker(engine)), + ) def test_complete_pause_flow_with_real_dependencies(workflow_case: WorkflowCase) -> None: diff --git a/api/tests/e2e_tests/containers/core/workflow/test_human_input_resume_node_execution.py b/api/tests/e2e_tests/containers/core/workflow/test_human_input_resume_node_execution.py index 5995ff8196cdcf..b94cd0e3fd1c2b 100644 --- a/api/tests/e2e_tests/containers/core/workflow/test_human_input_resume_node_execution.py +++ b/api/tests/e2e_tests/containers/core/workflow/test_human_input_resume_node_execution.py @@ -7,12 +7,8 @@ from core.app.app_config.entities import WorkflowUIBasedAppConfig from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity -from core.repositories.human_input_repository import HumanInputFormRepository, HumanInputFormRepositoryImpl from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository -from core.workflow.nodes.human_input.callback import ( - DifyHITLCallback, -) from core.workflow.system_variables import build_system_variables from enums.human_input import HumanInputFormStatus from graphon.engine import Engine @@ -31,11 +27,16 @@ from models.account import AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient +from models.human_input_contracts import HumanInputFormRepository from models.human_input_entities import HumanInputNodeData, UserActionConfig from models.model import App, AppMode, IconType from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowRun from models.workflow import WorkflowType as ModelWorkflowType -from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer +from repositories.human_input.form_repository import HumanInputFormRepositoryImpl +from services.workflow.execution.adapters.human_input import ( + DifyHITLCallback, +) +from services.workflow.execution.adapters.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer from services.workflow_run_index import WorkflowRunIndex from tests.workflow_test_utils import build_test_graph_init_params @@ -303,7 +304,10 @@ def test_resume_human_input_does_not_create_duplicate_node_execution(self): user_id=self.account.id, ) form_repository = HumanInputFormRepositoryImpl( - tenant_id=self.tenant.id, app_id=self.app.id, workflow_execution_id=execution_id + tenant_id=self.tenant.id, + app_id=self.app.id, + workflow_execution_id=execution_id, + sessions=sessionmaker(bind=self.session.get_bind()), ) paused_graph = _build_graph( runtime_state, diff --git a/api/tests/e2e_tests/containers/services/dataset_collection_binding.py b/api/tests/e2e_tests/containers/services/dataset_collection_binding.py index 037282a7d4e1f9..71fd59ea67f61c 100644 --- a/api/tests/e2e_tests/containers/services/dataset_collection_binding.py +++ b/api/tests/e2e_tests/containers/services/dataset_collection_binding.py @@ -1,7 +1,7 @@ """ -Comprehensive unit tests for DatasetCollectionBindingService. +Comprehensive unit tests for DatasetCollectionBindingRepository. -This module contains extensive unit tests for the DatasetCollectionBindingService class, +This module contains extensive unit tests for the DatasetCollectionBindingRepository class, which handles dataset collection binding operations for vector database collections. """ @@ -13,7 +13,7 @@ from models.dataset import DatasetCollectionBinding from models.enums import CollectionBindingType -from services.knowledge.dataset_service import DatasetCollectionBindingService +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository class DatasetCollectionBindingTestDataFactory: @@ -58,9 +58,9 @@ def create_collection_binding( return binding -class TestDatasetCollectionBindingServiceGetBinding: +class TestDatasetCollectionBindingRepositoryGetBinding: """ - Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding method. + Comprehensive unit tests for DatasetCollectionBindingRepository.get_dataset_collection_binding method. This test class covers the main collection binding retrieval/creation functionality, including various provider/model combinations, collection types, and edge cases. @@ -87,7 +87,7 @@ def test_get_dataset_collection_binding_existing_binding_success(self, db_sessio ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -108,7 +108,7 @@ def test_get_dataset_collection_binding_create_new_binding_success(self, db_sess collection_type = CollectionBindingType.DATASET # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -127,7 +127,7 @@ def test_get_dataset_collection_binding_different_collection_type(self, db_sessi collection_type = "custom_type" # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers, collection_type=collection_type ) @@ -143,7 +143,7 @@ def test_get_dataset_collection_binding_default_collection_type(self, db_session model_name = "text-embedding-ada-002" # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( provider_name, model_name, session=db_session_with_containers ) @@ -164,7 +164,7 @@ def test_get_dataset_collection_binding_different_provider_model_combination( ] # Act - results = list(starmap(DatasetCollectionBindingService.get_dataset_collection_binding, combinations)) + results = list(starmap(DatasetCollectionBindingRepository.get_dataset_collection_binding, combinations)) # Assert assert len(results) == 3 @@ -173,9 +173,9 @@ def test_get_dataset_collection_binding_different_provider_model_combination( assert result.model_name == model -class TestDatasetCollectionBindingServiceGetBindingByIdAndType: +class TestDatasetCollectionBindingRepositoryGetBindingByIdAndType: """ - Comprehensive unit tests for DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type method. + Verify collection binding lookup by ID and type through the shared repository. This test class covers retrieval of specific collection bindings by ID and type, including successful retrieval and error handling for missing bindings. @@ -193,7 +193,7 @@ def test_get_dataset_collection_binding_by_id_and_type_success(self, db_session_ ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type=CollectionBindingType.DATASET ) @@ -211,7 +211,7 @@ def test_get_dataset_collection_binding_by_id_and_type_not_found_error(self, db_ # Act & Assert with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( non_existent_id, session=db_session_with_containers, collection_type=CollectionBindingType.DATASET ) @@ -229,7 +229,7 @@ def test_get_dataset_collection_binding_by_id_and_type_different_collection_type ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type="custom_type" ) @@ -251,7 +251,7 @@ def test_get_dataset_collection_binding_by_id_and_type_default_collection_type( ) # Act - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers ) @@ -272,6 +272,6 @@ def test_get_dataset_collection_binding_by_id_and_type_wrong_type_error(self, db # Act & Assert with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers, collection_type="wrong_type" ) diff --git a/api/tests/e2e_tests/containers/services/test_app_dsl_service.py b/api/tests/e2e_tests/containers/services/test_app_dsl_service.py index 2e9ed4234847d9..313e2b936c41d6 100644 --- a/api/tests/e2e_tests/containers/services/test_app_dsl_service.py +++ b/api/tests/e2e_tests/containers/services/test_app_dsl_service.py @@ -4,7 +4,7 @@ import json from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, create_autospec, patch from uuid import uuid4 import pytest @@ -19,6 +19,8 @@ TRIGGER_SCHEDULE_NODE_TYPE, TRIGGER_WEBHOOK_NODE_TYPE, ) +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_app_dsl_service from extensions.ext_redis import redis_client from graphon.enums import BuiltinNodeTypes from models import Account, App, AppMode @@ -31,16 +33,15 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig from models.model import AppModelConfig, IconType from models.workflow import Workflow, WorkflowType +from repositories.app.creation_records import create_site_record from services import app_dsl_service from services.agent.dsl_entities import AGENT_PACKAGE_REF_KEY, make_portable_agent_package from services.agent.dsl_service import AgentDslService -from services.app_creation_records import create_site_record from services.app_dsl_service import ( CHECK_DEPENDENCIES_REDIS_KEY_PREFIX, CURRENT_DSL_VERSION, @@ -55,7 +56,9 @@ ) from services.app_service import AppService, CreateAppParams from services.dsl_version import check_version_compatibility +from services.entities.dsl_entities import AppDslOverwriteStore from services.errors.app import WorkflowNotFoundError +from services.workflow.draft_service import WorkflowDraftService from tests.e2e_tests.containers.helpers import accounts as account_fixtures from tests.e2e_tests.containers.helpers import generate_valid_password @@ -124,6 +127,7 @@ def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" with ( patch("services.app_dsl_service.WorkflowService") as mock_workflow_service, + patch("extensions.application_services.workflow.build_workflow_draft_service") as mock_drafts, patch("services.app_dsl_service.DependenciesAnalysisService") as mock_dependencies_service, patch("services.app_dsl_service.app_was_created") as mock_app_was_created, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, @@ -133,7 +137,7 @@ def mock_external_service_dependencies(self): mock_workflow_service.get_draft_workflow.return_value = None mock_workflow_service.get_published_workflow_by_id.return_value = None mock_workflow_service.return_value.get_draft_workflow.return_value = None - mock_workflow_service.return_value.sync_draft_workflow.return_value = MagicMock() + mock_drafts.return_value.sync.return_value = None mock_dependencies_service.generate_latest_dependencies.return_value = [] mock_dependencies_service.get_leaked_dependencies.return_value = [] mock_dependencies_service.generate_dependencies.return_value = [] @@ -151,6 +155,7 @@ def mock_external_service_dependencies(self): mock_enterprise_service.WebAppAuth.cleanup_webapp.return_value = None yield { "workflow_service": mock_workflow_service, + "drafts": mock_drafts, "dependencies_service": mock_dependencies_service, "app_was_created": mock_app_was_created, "model_manager": mock_model_manager, @@ -240,7 +245,7 @@ def test_check_version_compatibility_equal_returns_completed(self): # ── Import: Validation ──────────────────────────────────────────── def test_import_app_invalid_import_mode_raises_value_error(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Invalid import_mode"): service.import_app( account=_account(), @@ -249,7 +254,7 @@ def test_import_app_invalid_import_mode_raises_value_error(self, db_session_with ) def test_import_app_missing_yaml_content(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -259,7 +264,7 @@ def test_import_app_missing_yaml_content(self, db_session_with_containers: Sessi assert "yaml_content is required" in result.error def test_import_app_missing_yaml_url(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -269,7 +274,7 @@ def test_import_app_missing_yaml_url(self, db_session_with_containers: Session): assert "yaml_url is required" in result.error def test_import_app_yaml_not_mapping_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -279,7 +284,7 @@ def test_import_app_yaml_not_mapping_returns_failed(self, db_session_with_contai assert "content must be a mapping" in result.error def test_import_app_version_not_str_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) yaml_content = _yaml_dump({"version": 1, "kind": "app", "app": {"name": "x", "mode": "workflow"}}) result = service.import_app( account=_account(), @@ -290,7 +295,7 @@ def test_import_app_version_not_str_returns_failed(self, db_session_with_contain assert "Invalid version type" in result.error def test_import_app_missing_app_data_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -307,7 +312,7 @@ def bad_safe_load(_content: str): monkeypatch.setattr(app_dsl_service.yaml, "safe_load", bad_safe_load) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -325,7 +330,7 @@ def test_import_app_unexpected_error_returns_failed( lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("oops")), ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -345,7 +350,7 @@ def test_import_app_yaml_url_fetch_error_returns_failed( lambda _method, _url, **_kw: (_ for _ in ()).throw(RuntimeError("boom")), ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -362,7 +367,7 @@ def test_import_app_yaml_url_empty_content_returns_failed( response.raise_for_status.return_value = None monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", lambda _method, _url, **_kw: response) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -379,7 +384,7 @@ def test_import_app_yaml_url_file_too_large_returns_failed( response.raise_for_status.return_value = None monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", lambda _method, _url, **_kw: response) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -406,7 +411,7 @@ def fake_make_request(method: str, url: str, **kwargs): monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", fake_make_request) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -437,7 +442,7 @@ def fake_make_request(method: str, url: str, **kwargs): monkeypatch.setattr(app_dsl_service.remote_fetcher, "make_request", fake_make_request) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_URL, @@ -450,7 +455,7 @@ def fake_make_request(method: str, url: str, **kwargs): # ── Import: App ID checks ──────────────────────────────────────── def test_import_app_app_id_not_found_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -466,7 +471,7 @@ def test_import_app_overwrite_only_allows_workflow_and_advanced_chat( app, account = self._create_test_app_and_account(db_session_with_containers, mock_external_service_dependencies) assert app.mode == "chat" - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=account, import_mode=ImportMode.YAML_CONTENT, @@ -474,12 +479,12 @@ def test_import_app_overwrite_only_allows_workflow_and_advanced_chat( app_id=app.id, ) assert result.status == ImportStatus.FAILED - assert "Only workflow or advanced chat apps" in result.error + assert result.error == "Only workflow or advanced chat DSLs can overwrite workflow Apps" # ── Import: Flow ────────────────────────────────────────────────── def test_import_app_pending_stores_import_info_in_redis(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, @@ -515,7 +520,7 @@ def test_import_app_completed_uses_declared_dependencies( } ] - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=account, import_mode=ImportMode.YAML_CONTENT, @@ -564,13 +569,6 @@ def test_import_app_legacy_versions_extract_dependencies( lambda *_args, **_kwargs: created_app, ) - draft_var_service = MagicMock() - monkeypatch.setattr( - app_dsl_service, - "WorkflowDraftVariableService", - lambda *args, **kwargs: draft_var_service, - ) - data: dict = { "version": "0.1.5", "kind": "app", @@ -581,19 +579,18 @@ def test_import_app_legacy_versions_extract_dependencies( else: data["model_config"] = {"model": {"provider": "openai"}} - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.import_app( account=_account(), import_mode=ImportMode.YAML_CONTENT, yaml_content=_yaml_dump(data), ) assert result.status == ImportStatus.COMPLETED_WITH_WARNINGS - draft_var_service.delete_app_workflow_variables.assert_called_once_with(app_id=created_app.id) # ── Confirm Import ──────────────────────────────────────────────── def test_confirm_import_expired_returns_failed(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=str(uuid4()), account=_account()) assert result.status == ImportStatus.FAILED assert "expired" in result.error @@ -629,7 +626,7 @@ def test_confirm_import_success_deletes_redis_key( lambda *_args, **_kwargs: created_app, ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.COMPLETED assert result.app_id == created_app.id @@ -640,7 +637,7 @@ def test_confirm_import_invalid_pending_data_type_returns_failed(self, db_sessio redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}" redis_client.setex(redis_key, IMPORT_INFO_REDIS_EXPIRY, "123") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.FAILED assert "validation error" in result.error @@ -650,14 +647,14 @@ def test_confirm_import_exception_returns_failed(self, db_session_with_container redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}" redis_client.setex(redis_key, IMPORT_INFO_REDIS_EXPIRY, "not-valid-json") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.confirm_import(import_id=import_id, account=_account()) assert result.status == ImportStatus.FAILED # ── Check Dependencies ──────────────────────────────────────────── def test_check_dependencies_returns_empty_when_no_redis_data(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) app_model = _app_record() result = service.check_dependencies(app_model=app_model) assert result.leaked_dependencies == [] @@ -688,7 +685,7 @@ def test_check_dependencies_calls_analysis_service( lambda *, tenant_id, dependencies: [dep], ) - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) result = service.check_dependencies(app_model=_app_record(id=app_id)) assert len(result.leaked_dependencies) == 1 @@ -704,14 +701,14 @@ def test_check_dependencies_with_real_app( mock_dependencies_json, ) - dsl_service = AppDslService(db_session_with_containers) + dsl_service = build_app_dsl_service(db_session_with_containers) result = dsl_service.check_dependencies(app_model=app) assert result.leaked_dependencies == [] # ── Create/Update App ───────────────────────────────────────────── def test_create_or_update_app_missing_mode_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="loss app mode"): service._create_or_update_app(app=None, data={"app": {}}, account=_account()) @@ -721,19 +718,7 @@ def test_create_or_update_app_existing_app_updates_fields( fixed_now = object() monkeypatch.setattr(app_dsl_service, "naive_utc_now", lambda: fixed_now) - workflow_service = MagicMock() - workflow_service.get_draft_workflow.return_value = None - monkeypatch.setattr(app_dsl_service, "WorkflowService", lambda: workflow_service) - monkeypatch.setattr( - app_dsl_service.variable_factory, - "build_environment_variable_from_mapping", - lambda _m: SimpleNamespace(kind="env"), - ) - monkeypatch.setattr( - app_dsl_service.variable_factory, - "build_conversation_variable_from_mapping", - lambda _m: SimpleNamespace(kind="conv"), - ) + drafts = create_autospec(WorkflowDraftService, instance=True, spec_set=True) app = _app_record( mode=AppMode.WORKFLOW, @@ -745,7 +730,11 @@ def test_create_or_update_app_existing_app_updates_fields( updated_by=None, updated_at=None, ) - service = AppDslService(db_session_with_containers) + service = AppDslService( + db_session_with_containers, + drafts=drafts, + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) updated = service._create_or_update_app( app=app, data={ @@ -769,10 +758,19 @@ def test_create_or_update_app_existing_app_updates_fields( assert app.icon_background == "#222222" assert app.updated_at is fixed_now + drafts.sync.assert_called_once() + context, owner, command = drafts.sync.call_args.args + assert context.account_id == _DEFAULT_ACCOUNT_ID + assert context.active_workspace_id == app.tenant_id + assert owner.id == app.id + assert command.graph == {"nodes": []} + assert command.check_hash is False + assert command.clear_debug_variables is True + def test_create_or_update_app_new_app_requires_tenant(self, db_session_with_containers: Session): account = _account() account._current_tenant = None - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Current tenant is not set"): service._create_or_update_app( app=None, @@ -788,7 +786,7 @@ def test_create_or_update_app_creates_workflow_app_and_saves_dependencies( mock_wf_svc = mock_external_service_dependencies["workflow_service"] mock_wf_svc.return_value.get_draft_workflow.return_value = MagicMock(unique_hash="uh") - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) deps = [ app_dsl_service.PluginDependency.model_validate( { @@ -812,13 +810,13 @@ def test_create_or_update_app_creates_workflow_app_and_saves_dependencies( assert app.tenant_id == account.current_tenant_id mock_external_service_dependencies["app_was_created"].send.assert_called_once() - mock_wf_svc.return_value.sync_draft_workflow.assert_called_once() + mock_external_service_dependencies["drafts"].return_value.sync.assert_called_once() stored = redis_client.get(f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app.id}") assert stored is not None def test_create_or_update_app_workflow_missing_workflow_data_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Missing workflow data"): service._create_or_update_app( app=_app_record(mode=AppMode.WORKFLOW), @@ -827,7 +825,7 @@ def test_create_or_update_app_workflow_missing_workflow_data_raises(self, db_ses ) def test_create_or_update_app_chat_requires_model_config(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Missing model_config"): service._create_or_update_app( app=_app_record(mode=AppMode.CHAT), @@ -842,7 +840,7 @@ def test_create_or_update_app_chat_creates_model_config_and_sends_event( app.app_model_config_id = None db_session_with_containers.commit() - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with patch("services.app_dsl_service.app_model_config_was_updated") as signal: service._create_or_update_app( app=app, @@ -858,7 +856,7 @@ def test_create_or_update_app_chat_creates_model_config_and_sends_event( assert signal.send.call_args.kwargs["session"] is db_session_with_containers def test_create_or_update_app_invalid_mode_raises(self, db_session_with_containers: Session): - service = AppDslService(db_session_with_containers) + service = build_app_dsl_service(db_session_with_containers) with pytest.raises(ValueError, match="Invalid app mode"): service._create_or_update_app( app=_app_record(mode=AppMode.RAG_PIPELINE), @@ -1209,7 +1207,7 @@ def test_agent_app_dsl_round_trip_creates_unpublished_imported_agent( mock_external_service_dependencies["app_was_created"].send.side_effect = ( lambda app, *, account, session, **_kwargs: create_site_record(app=app, account=account, session=session) ) - result = AppDslService(db_session_with_containers).import_app( + result = build_app_dsl_service(db_session_with_containers).import_app( account=account, import_mode=ImportMode.YAML_CONTENT, yaml_content=yaml_content, diff --git a/api/tests/e2e_tests/containers/services/test_app_generate_service.py b/api/tests/e2e_tests/containers/services/test_app_generate_service.py index 9fdc4b6129f41b..1b35be43348f37 100644 --- a/api/tests/e2e_tests/containers/services/test_app_generate_service.py +++ b/api/tests/e2e_tests/containers/services/test_app_generate_service.py @@ -1,12 +1,16 @@ +from __future__ import annotations + import json import uuid -from typing import Literal +from collections.abc import Generator +from types import MethodType +from typing import TYPE_CHECKING, Literal from unittest.mock import ANY, MagicMock, patch import pytest import sqlalchemy as sa from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.app.entities.app_invoke_entities import InvokeFrom from enums import DeploymentEdition @@ -18,18 +22,41 @@ from services.app_generate_service import AppGenerateService from services.billing_client import billing from services.errors.app import WorkflowIdFormatError, WorkflowNotFoundError +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.e2e_tests.containers.helpers import generate_valid_password +if TYPE_CHECKING: + from extensions.application_services.workflow import WorkflowExecutionDependencies + + +@pytest.fixture +def workflow_runtime(db_session_with_containers: Session) -> WorkflowExecutionDependencies: + from extensions.application_services.workflow import build_workflow_execution_dependencies + + return build_workflow_execution_dependencies( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + class TestAppGenerateService: """Integration tests for AppGenerateService using testcontainers.""" @pytest.fixture - def mock_external_service_dependencies(self): + def mock_external_service_dependencies(self) -> Generator[dict[str, MagicMock]]: """Mock setup for external service dependencies.""" with ( patch("services.billing_service.BillingService", autospec=True) as mock_billing_service, - patch("services.app_generate_service.WorkflowService", autospec=True) as mock_workflow_service, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_draft_workflow", autospec=True + ) as mock_draft_workflow, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_published_workflow", + autospec=True, + ) as mock_published_workflow, + patch( + "repositories.workflow.definition_repository.WorkflowDefinitionStore.get_published_workflow_by_id", + autospec=True, + ) as mock_workflow_by_id, patch("services.app_generate_service.RateLimit", autospec=True) as mock_rate_limit, patch("services.app_generate_service.CompletionAppGenerator", autospec=True) as mock_completion_generator, patch("services.app_generate_service.ChatAppGenerator", autospec=True) as mock_chat_generator, @@ -38,9 +65,10 @@ def mock_external_service_dependencies(self): "services.app_generate_service.AdvancedChatAppGenerator", autospec=True ) as mock_advanced_chat_generator, patch("services.app_generate_service.WorkflowAppGenerator", autospec=True) as mock_workflow_generator, + patch("services.app_generate_service.convert_to_event_stream", autospec=True) as mock_event_stream, patch( - "services.app_generate_service.MessageBasedAppGenerator", autospec=True - ) as mock_message_based_generator, + "services.app_generate_service.WorkflowEventStream.retrieve_events", autospec=True + ) as mock_retrieve_events, patch( "services.account.login_adapters.SystemFeatureService", autospec=True ) as mock_account_feature_service, @@ -56,8 +84,7 @@ def mock_external_service_dependencies(self): available=99, reserved=0, refunded=0 ) - # Setup default mock returns for workflow service - mock_workflow_service_instance = mock_workflow_service.return_value + # Supply persisted workflow reads while keeping execution selection intact. graph = json.dumps( { "nodes": [ @@ -68,10 +95,10 @@ def mock_external_service_dependencies(self): } ) published_workflow = Workflow(id=str(uuid.uuid4()), version="1", graph=graph) - mock_workflow_service_instance.get_published_workflow.return_value = published_workflow + mock_published_workflow.return_value = published_workflow draft_workflow = Workflow(id=str(uuid.uuid4()), version="draft", graph=graph) - mock_workflow_service_instance.get_draft_workflow.return_value = draft_workflow - mock_workflow_service_instance.get_published_workflow_by_id.return_value = published_workflow + mock_draft_workflow.return_value = draft_workflow + mock_workflow_by_id.return_value = published_workflow # Setup default mock returns for rate limiting mock_rate_limit_instance = mock_rate_limit.return_value @@ -82,23 +109,17 @@ def mock_external_service_dependencies(self): # Setup default mock returns for app generators mock_completion_generator_instance = mock_completion_generator.return_value mock_completion_generator_instance.generate.return_value = ["completion_response"] - mock_completion_generator.convert_to_event_stream.return_value = ["completion_stream"] mock_chat_generator_instance = mock_chat_generator.return_value mock_chat_generator_instance.generate.return_value = ["chat_response"] - mock_chat_generator.convert_to_event_stream.return_value = ["chat_stream"] mock_agent_chat_generator_instance = mock_agent_chat_generator.return_value mock_agent_chat_generator_instance.generate.return_value = ["agent_chat_response"] - mock_agent_chat_generator.convert_to_event_stream.return_value = ["agent_chat_stream"] mock_advanced_chat_generator_instance = mock_advanced_chat_generator.return_value mock_advanced_chat_generator_instance.generate.return_value = ["advanced_chat_response"] mock_advanced_chat_generator_instance.single_iteration_generate.return_value = ["single_iteration_response"] mock_advanced_chat_generator_instance.single_loop_generate.return_value = ["single_loop_response"] - mock_advanced_chat_generator.retrieve_events.return_value = ["advanced_chat_events"] - mock_advanced_chat_generator_instance.convert_to_event_stream.return_value = ["advanced_chat_stream"] - mock_advanced_chat_generator.convert_to_event_stream.return_value = ["advanced_chat_stream"] mock_workflow_generator_instance = mock_workflow_generator.return_value mock_workflow_generator_instance.generate.return_value = ["workflow_response"] @@ -106,9 +127,8 @@ def mock_external_service_dependencies(self): "workflow_single_iteration_response" ] mock_workflow_generator_instance.single_loop_generate.return_value = ["workflow_single_loop_response"] - mock_workflow_generator.convert_to_event_stream.return_value = ["workflow_stream"] - - mock_message_based_generator.retrieve_events.return_value = ["workflow_events"] + mock_event_stream.side_effect = lambda events: events + mock_retrieve_events.return_value = ["workflow_events"] # Setup default mock returns for account service mock_account_feature_service.is_registration_allowed.return_value = True @@ -128,14 +148,17 @@ def mock_external_service_dependencies(self): yield { "billing_service": mock_billing_service, - "workflow_service": mock_workflow_service, + "get_draft_workflow": mock_draft_workflow, + "get_published_workflow": mock_published_workflow, + "get_published_workflow_by_id": mock_workflow_by_id, "rate_limit": mock_rate_limit, "completion_generator": mock_completion_generator, "chat_generator": mock_chat_generator, "agent_chat_generator": mock_agent_chat_generator, "advanced_chat_generator": mock_advanced_chat_generator, "workflow_generator": mock_workflow_generator, - "message_based_generator": mock_message_based_generator, + "event_stream": mock_event_stream, + "retrieve_events": mock_retrieve_events, "account_feature_service": mock_account_feature_service, "dify_config": mock_dify_config, "quota_dify_config": mock_quota_dify_config, @@ -229,7 +252,12 @@ def _create_test_workflow(self, db_session_with_containers: Session, app: App): return workflow def test_generate_completion_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for completion mode app. @@ -244,12 +272,14 @@ def test_generate_completion_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -260,10 +290,18 @@ def test_generate_completion_mode_success( mock_external_service_dependencies["rate_limit"].return_value.generate.assert_called_once() # Verify completion generator was called - mock_external_service_dependencies["completion_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["completion_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["completion_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) - def test_generate_chat_mode_success(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_chat_mode_success( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test successful generation for chat mode app. """ @@ -277,23 +315,31 @@ def test_generate_chat_mode_success(self, db_session_with_containers: Session, m # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] # Verify chat generator was called - mock_external_service_dependencies["chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["chat_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["chat_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_agent_chat_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for agent chat mode app. @@ -308,23 +354,42 @@ def test_generate_agent_chat_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] # Verify agent chat generator was called - mock_external_service_dependencies["agent_chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() + generator_factory = mock_external_service_dependencies["agent_chat_generator"] + generator_factory.assert_called_once() + constructor_args = generator_factory.call_args.kwargs + assert constructor_args["tool_invoker"] is workflow_runtime.agent_tool_invoker + draft_variable_saver = constructor_args["draft_variable_saver"] + expected_saver = workflow_variables.saver_factory + assert isinstance(draft_variable_saver, MethodType) + assert isinstance(expected_saver, MethodType) + assert draft_variable_saver.__self__ is expected_saver.__self__ + assert draft_variable_saver.__func__ is expected_saver.__func__ + assert constructor_args["workflow_runtime"] is workflow_runtime + generator = generator_factory.return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_advanced_chat_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for advanced chat mode app. @@ -339,23 +404,31 @@ def test_generate_advanced_chat_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - # Verify advanced chat generator was called - mock_external_service_dependencies["advanced_chat_generator"].retrieve_events.assert_called_once() - mock_external_service_dependencies["advanced_chat_generator"].convert_to_event_stream.assert_called_once() + # Streaming workflows read the shared event stream instead of constructing a synchronous generator. + events = mock_external_service_dependencies["retrieve_events"] + events.assert_called_once_with(ANY, ANY, on_subscribe=ANY) + mock_external_service_dependencies["event_stream"].assert_called_once_with(events.return_value) def test_generate_workflow_mode_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful generation for workflow mode app. @@ -370,23 +443,31 @@ def test_generate_workflow_mode_success( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - # Verify workflow generator was called - mock_external_service_dependencies["message_based_generator"].retrieve_events.assert_called_once() - mock_external_service_dependencies["workflow_generator"].convert_to_event_stream.assert_called_once() + # Streaming workflows read the shared event stream instead of constructing a synchronous generator. + events = mock_external_service_dependencies["retrieve_events"] + events.assert_called_once_with(ANY, ANY, on_subscribe=ANY) + mock_external_service_dependencies["event_stream"].assert_called_once_with(events.return_value) def test_generate_with_specific_workflow_id( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with a specific workflow ID. @@ -407,24 +488,29 @@ def test_generate_with_specific_workflow_id( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - # Verify workflow service was called with specific workflow ID - mock_external_service_dependencies[ - "workflow_service" - ].return_value.get_published_workflow_by_id.assert_called_once() + # Verify the repository selected the specific workflow ID + mock_external_service_dependencies["get_published_workflow_by_id"].assert_called_once() def test_generate_with_debugger_invoke_from( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with debugger invoke from. @@ -439,22 +525,29 @@ def test_generate_with_debugger_invoke_from( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] # Verify draft workflow was fetched for debugger - mock_external_service_dependencies["workflow_service"].return_value.get_draft_workflow.assert_called_once() + mock_external_service_dependencies["get_draft_workflow"].assert_called_once() def test_generate_with_non_streaming_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with non-streaming mode. @@ -469,12 +562,14 @@ def test_generate_with_non_streaming_mode( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -483,7 +578,14 @@ def test_generate_with_non_streaming_mode( # Verify rate limit exit was called for non-streaming mode mock_external_service_dependencies["rate_limit"].return_value.exit.assert_called_once() - def test_generate_with_end_user(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_with_end_user( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test generation with EndUser instead of Account. """ @@ -511,19 +613,26 @@ def test_generate_with_end_user(self, db_session_with_containers: Session, mock_ # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=end_user, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] def test_generate_in_cloud_sandbox_plan( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation in the Cloud edition with a sandbox plan. @@ -543,12 +652,14 @@ def test_generate_in_cloud_sandbox_plan( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -560,7 +671,12 @@ def test_generate_in_cloud_sandbox_plan( billing.quota_commit.assert_called_once() def test_generate_with_invalid_app_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with invalid app mode. @@ -582,16 +698,23 @@ def test_generate_with_invalid_app_mode( # StatementError (from EnumText validation during autoflush) with pytest.raises((ValueError, sa.exc.StatementError)): AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) def test_generate_with_workflow_id_format_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with invalid workflow ID format. @@ -611,19 +734,26 @@ def test_generate_with_workflow_id_format_error( # Execute the method under test and expect WorkflowIdFormatError with pytest.raises(WorkflowIdFormatError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Invalid workflow_id format" in str(exc_info.value) def test_generate_with_workflow_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not found. @@ -635,10 +765,8 @@ def test_generate_with_workflow_not_found_error( workflow_id = str(uuid.uuid4()) - # Setup workflow service to return None (workflow not found) - mock_external_service_dependencies[ - "workflow_service" - ].return_value.get_published_workflow_by_id.return_value = None + # Return no persisted workflow for the requested ID + mock_external_service_dependencies["get_published_workflow_by_id"].return_value = None # Setup test arguments args = { @@ -650,19 +778,26 @@ def test_generate_with_workflow_not_found_error( # Execute the method under test and expect WorkflowNotFoundError with pytest.raises(WorkflowNotFoundError) as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert f"Workflow not found with id: {workflow_id}" in str(exc_info.value) def test_generate_with_workflow_not_initialized_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not initialized for debugger. @@ -673,7 +808,7 @@ def test_generate_with_workflow_not_initialized_error( ) # Setup workflow service to return None (workflow not initialized) - mock_external_service_dependencies["workflow_service"].return_value.get_draft_workflow.return_value = None + mock_external_service_dependencies["get_draft_workflow"].return_value = None # Setup test arguments args = {"inputs": {"query": fake.text(max_nb_chars=50)}, "response_mode": "streaming"} @@ -681,19 +816,26 @@ def test_generate_with_workflow_not_initialized_error( # Execute the method under test and expect ValueError with pytest.raises(ValueError, match="Workflow not initialized") as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Workflow not initialized" in str(exc_info.value) def test_generate_with_workflow_not_published_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation when workflow is not published for non-debugger. @@ -704,7 +846,7 @@ def test_generate_with_workflow_not_published_error( ) # Setup workflow service to return None (workflow not published) - mock_external_service_dependencies["workflow_service"].return_value.get_published_workflow.return_value = None + mock_external_service_dependencies["get_published_workflow"].return_value = None # Setup test arguments args = {"inputs": {"query": fake.text(max_nb_chars=50)}, "response_mode": "streaming"} @@ -712,19 +854,26 @@ def test_generate_with_workflow_not_published_error( # Execute the method under test and expect ValueError with pytest.raises(ValueError, match="Workflow not published") as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Workflow not published" in str(exc_info.value) def test_generate_single_iteration_advanced_chat_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single iteration generation for advanced chat mode. @@ -739,16 +888,18 @@ def test_generate_single_iteration_advanced_chat_success( # Execute the method under test result = AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["single_iteration_response"] # Verify advanced chat generator was called mock_external_service_dependencies[ @@ -756,7 +907,12 @@ def test_generate_single_iteration_advanced_chat_success( ].return_value.single_iteration_generate.assert_called_once() def test_generate_single_iteration_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single iteration generation for workflow mode. @@ -771,16 +927,18 @@ def test_generate_single_iteration_workflow_success( # Execute the method under test result = AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["workflow_single_iteration_response"] # Verify workflow generator was called mock_external_service_dependencies[ @@ -788,7 +946,12 @@ def test_generate_single_iteration_workflow_success( ].return_value.single_iteration_generate.assert_called_once() def test_generate_single_iteration_invalid_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test single iteration generation with invalid app mode. @@ -804,19 +967,26 @@ def test_generate_single_iteration_invalid_mode( # Execute the method under test and expect ValueError with pytest.raises(ValueError, match="Invalid app mode") as exc_info: AppGenerateService.generate_single_iteration( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message assert "Invalid app mode" in str(exc_info.value) def test_generate_single_loop_advanced_chat_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single loop generation for advanced chat mode. @@ -831,16 +1001,18 @@ def test_generate_single_loop_advanced_chat_success( # Execute the method under test result = AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["single_loop_response"] # Verify advanced chat generator was called mock_external_service_dependencies[ @@ -848,7 +1020,12 @@ def test_generate_single_loop_advanced_chat_success( ].return_value.single_loop_generate.assert_called_once() def test_generate_single_loop_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test successful single loop generation for workflow mode. @@ -863,22 +1040,29 @@ def test_generate_single_loop_workflow_success( # Execute the method under test result = AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result - assert result == ["advanced_chat_stream"] + assert result == ["workflow_single_loop_response"] # Verify workflow generator was called mock_external_service_dependencies["workflow_generator"].return_value.single_loop_generate.assert_called_once() def test_generate_single_loop_invalid_mode( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test single loop generation with invalid app mode. @@ -894,12 +1078,14 @@ def test_generate_single_loop_invalid_mode( # Execute the method under test and expect ValueError with pytest.raises(ValueError, match="Invalid app mode") as exc_info: AppGenerateService.generate_single_loop( + runtime=workflow_runtime, + workflow=mock_external_service_dependencies["get_draft_workflow"].return_value, app_model=app, user=account, node_id=node_id, args=args, streaming=True, - session=db_session_with_containers, + variables=workflow_variables, ) # Verify error message @@ -967,7 +1153,12 @@ def test_get_max_active_requests_with_zero_limits( assert result == 100 # dify_config.APP_MAX_ACTIVE_REQUESTS def test_generate_with_exception_cleanup( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test that rate limit exit is called when an exception occurs. @@ -988,12 +1179,14 @@ def test_generate_with_exception_cleanup( # Execute the method under test and expect exception with pytest.raises(Exception, match="Test exception") as exc_info: AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify exception message @@ -1003,7 +1196,12 @@ def test_generate_with_exception_cleanup( mock_external_service_dependencies["rate_limit"].return_value.exit.assert_called_once() def test_generate_with_agent_mode_detection( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with agent mode detection based on app configuration. @@ -1021,23 +1219,31 @@ def test_generate_with_agent_mode_detection( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] # Verify agent chat generator was called instead of regular chat generator - mock_external_service_dependencies["agent_chat_generator"].return_value.generate.assert_called_once() - mock_external_service_dependencies["agent_chat_generator"].convert_to_event_stream.assert_called_once() + generator = mock_external_service_dependencies["agent_chat_generator"].return_value + generator.generate.assert_called_once() + mock_external_service_dependencies["event_stream"].assert_called_once_with(generator.generate.return_value) def test_generate_with_different_invoke_from_values( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, ): """ Test generation with different invoke from values. @@ -1061,18 +1267,27 @@ def test_generate_with_different_invoke_from_values( # Execute the method under test result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=invoke_from, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result assert result == ["test_response"] - def test_generate_with_complex_args(self, db_session_with_containers: Session, mock_external_service_dependencies): + def test_generate_with_complex_args( + self, + workflow_runtime, + db_session_with_containers: Session, + mock_external_service_dependencies, + *, + workflow_variables: WorkflowExecutionVariables, + ): """ Test generation with complex arguments including files and external trace ID. """ @@ -1104,12 +1319,14 @@ def test_generate_with_complex_args(self, db_session_with_containers: Session, m mock_exec_params.new.return_value = mock_payload result = AppGenerateService.generate( + runtime=workflow_runtime, app_model=app, user=account, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=True, session=db_session_with_containers, + variables=workflow_variables, ) # Verify the result @@ -1121,7 +1338,7 @@ def test_generate_with_complex_args(self, db_session_with_containers: Session, m assert call_kwargs["args"] == args # Verify workflow streaming event retrieval was used - mock_external_service_dependencies["message_based_generator"].retrieve_events.assert_called_once_with( + mock_external_service_dependencies["retrieve_events"].assert_called_once_with( ANY, mock_payload.workflow_run_id, on_subscribe=ANY, diff --git a/api/tests/e2e_tests/containers/services/test_conversation_variable_updater.py b/api/tests/e2e_tests/containers/services/test_conversation_variable_updater.py index 638a962f18bd9f..b17450d1b85f89 100644 --- a/api/tests/e2e_tests/containers/services/test_conversation_variable_updater.py +++ b/api/tests/e2e_tests/containers/services/test_conversation_variable_updater.py @@ -1,4 +1,4 @@ -"""Testcontainers integration tests for ConversationVariableUpdater.""" +"""Testcontainers integration tests for WorkflowConversationVariableRepository.""" from uuid import uuid4 @@ -8,10 +8,13 @@ from extensions.ext_database import db from graphon.variables import StringVariable from models.workflow import ConversationVariable -from services.conversation_variable_updater import ConversationVariableNotFoundError, ConversationVariableUpdater +from repositories.workflow.conversation_variable_repository import ( + ConversationVariableNotFoundError, + WorkflowConversationVariableRepository, +) -class TestConversationVariableUpdater: +class TestWorkflowConversationVariableRepository: def _create_conversation_variable( self, db_session_with_containers: Session, @@ -38,7 +41,7 @@ def test_should_update_conversation_variable_data_and_commit(self, db_session_wi ) updated_variable = StringVariable(id=variable.id, name="topic", value="new value") - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=db.engine)) updater.update(conversation_id=conversation_id, variable=updated_variable) @@ -50,14 +53,7 @@ def test_should_update_conversation_variable_data_and_commit(self, db_session_wi def test_should_raise_not_found_when_variable_missing(self, db_session_with_containers: Session): conversation_id = str(uuid4()) variable = StringVariable(id=str(uuid4()), name="topic", value="value") - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=db.engine)) with pytest.raises(ConversationVariableNotFoundError, match="conversation variable not found in the database"): updater.update(conversation_id=conversation_id, variable=variable) - - def test_should_do_nothing_when_flush_is_called(self, db_session_with_containers: Session): - updater = ConversationVariableUpdater(sessionmaker(bind=db.engine)) - - result = updater.flush() - - assert result is None diff --git a/api/tests/e2e_tests/containers/services/test_dataset_service_document.py b/api/tests/e2e_tests/containers/services/test_dataset_service_document.py index e221eaf4be3bb0..1ef9540116e8cb 100644 --- a/api/tests/e2e_tests/containers/services/test_dataset_service_document.py +++ b/api/tests/e2e_tests/containers/services/test_dataset_service_document.py @@ -7,7 +7,6 @@ import pytest from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, NotFound from core.rag.index_processor.constant.index_type import IndexStructureType from extensions.storage.storage_type import StorageType @@ -17,7 +16,10 @@ from models.enums import CreatorUserRole, DataSourceType, DocumentCreatedFrom, IndexingStatus from models.model import UploadFile from repositories.knowledge.dataset_read_repository import get_dataset_doc_form +from repositories.knowledge.document_repository import next_document_position from services.errors.base import NoPermissionError +from services.errors.dataset import DatasetNotFoundError +from services.errors.document import DocumentAccessDeniedError, DocumentNotFoundError, DocumentSourceNotFoundError from services.knowledge.dataset_service import DocumentService from services.knowledge.resource_scope import DatasetRef @@ -239,7 +241,7 @@ def test_get_upload_file_id_for_upload_file_document_rejects_invalid_source_type data_source_info={"url": "https://example.com"}, ) - with pytest.raises(NotFound, match="invalid source"): + with pytest.raises(DocumentSourceNotFoundError, match="invalid source"): DocumentService._get_upload_file_id_for_upload_file_document( document, invalid_source_message="invalid source", @@ -257,7 +259,7 @@ def test_get_upload_file_id_for_upload_file_document_rejects_missing_upload_file data_source_info={}, ) - with pytest.raises(NotFound, match="missing file"): + with pytest.raises(DocumentSourceNotFoundError, match="missing file"): DocumentService._get_upload_file_id_for_upload_file_document( document, invalid_source_message="invalid source", @@ -293,7 +295,7 @@ def test_get_upload_file_for_upload_file_document_raises_when_file_service_retur ) with patch("services.knowledge.dataset_service.FileService.get_upload_files_by_ids", return_value={}): - with pytest.raises(NotFound, match="Uploaded file not found"): + with pytest.raises(DocumentSourceNotFoundError, match="Uploaded file not found"): DocumentService._get_upload_file_for_upload_file_document(document, session=db_session_with_containers) @@ -320,7 +322,7 @@ def test_get_upload_files_by_document_id_for_zip_download_raises_for_missing_doc ): dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[str(uuid4())], @@ -345,7 +347,7 @@ def test_get_upload_files_by_document_id_for_zip_download_hides_cross_tenant_doc data_source_info={"upload_file_id": upload_file.id}, ) - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[document.id], @@ -364,7 +366,7 @@ def test_get_upload_files_by_document_id_for_zip_download_rejects_missing_upload data_source_info={"upload_file_id": str(uuid4())}, ) - with pytest.raises(NotFound, match="Only uploaded-file documents can be downloaded as ZIP"): + with pytest.raises(DocumentSourceNotFoundError, match="Only uploaded-file documents can be downloaded as ZIP"): DocumentService._get_upload_files_by_document_id_for_zip_download( dataset_id=dataset.id, document_ids=[document.id], @@ -416,7 +418,7 @@ def test_prepare_document_batch_download_zip_raises_not_found_for_missing_datase current_user_mock, flask_app_with_containers, db_session_with_containers: Session ): with flask_app_with_containers.app_context(): - with pytest.raises(NotFound, match="Dataset not found"): + with pytest.raises(DatasetNotFoundError, match="Dataset not found"): DocumentService.prepare_document_batch_download_zip( dataset_id=str(uuid4()), document_ids=[str(uuid4())], @@ -440,7 +442,7 @@ def test_prepare_document_batch_download_zip_translates_permission_error_to_forb "services.knowledge.dataset_service.DatasetService.check_dataset_permission", side_effect=NoPermissionError("denied"), ): - with pytest.raises(Forbidden, match="denied"): + with pytest.raises(DocumentAccessDeniedError, match="denied"): DocumentService.prepare_document_batch_download_zip( dataset_id=dataset.id, document_ids=[], @@ -692,10 +694,10 @@ def test_get_documents_position_returns_next_position_when_documents_exist(db_se dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) DocumentServiceIntegrationFactory.create_document(db_session_with_containers, dataset=dataset, position=3) - assert DocumentService.get_documents_position(dataset.id, session=db_session_with_containers) == 4 + assert next_document_position(dataset.id, session=db_session_with_containers) == 4 def test_get_documents_position_defaults_to_one_when_dataset_is_empty(db_session_with_containers: Session): dataset = DocumentServiceIntegrationFactory.create_dataset(db_session_with_containers) - assert DocumentService.get_documents_position(dataset.id, session=db_session_with_containers) == 1 + assert next_document_position(dataset.id, session=db_session_with_containers) == 1 diff --git a/api/tests/e2e_tests/containers/services/test_dataset_service_permissions.py b/api/tests/e2e_tests/containers/services/test_dataset_service_permissions.py index 39a4371d5eb2a2..539219507a40b9 100644 --- a/api/tests/e2e_tests/containers/services/test_dataset_service_permissions.py +++ b/api/tests/e2e_tests/containers/services/test_dataset_service_permissions.py @@ -21,8 +21,9 @@ DatasetPermissionEnum, ) from models.enums import DataSourceType +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.errors.base import NoPermissionError -from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, DatasetService +from services.knowledge.dataset_service import DatasetPermissionService, DatasetService from services.knowledge.resource_scope import DatasetRef from tests.helpers.unit.config_override import config_overrides_context @@ -449,7 +450,7 @@ def test_get_dataset_auto_disable_logs_returns_recent_document_ids(self, db_sess assert len(result["document_ids"]) == 2 -class TestDatasetCollectionBindingServiceIntegration: +class TestDatasetCollectionBindingRepositoryIntegration: def test_get_dataset_collection_binding_returns_existing_binding(self, db_session_with_containers: Session): binding = DatasetPermissionIntegrationFactory.create_collection_binding( db_session_with_containers, @@ -457,14 +458,14 @@ def test_get_dataset_collection_binding_returns_existing_binding(self, db_sessio model_name="model", ) - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( "provider", "model", session=db_session_with_containers ) assert result.id == binding.id def test_get_dataset_collection_binding_creates_binding_when_missing(self, db_session_with_containers: Session): - result = DatasetCollectionBindingService.get_dataset_collection_binding( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding( "provider", "missing-model", session=db_session_with_containers ) @@ -480,7 +481,7 @@ def test_get_dataset_collection_binding_by_id_and_type_raises_when_missing( ): with flask_app_with_containers.app_context(): with pytest.raises(ValueError, match="Dataset collection binding not found"): - DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( str(uuid4()), session=db_session_with_containers ) @@ -491,7 +492,7 @@ def test_get_dataset_collection_binding_by_id_and_type_returns_binding(self, db_ model_name="model", ) - result = DatasetCollectionBindingService.get_dataset_collection_binding_by_id_and_type( + result = DatasetCollectionBindingRepository.get_dataset_collection_binding_by_id_and_type( binding.id, session=db_session_with_containers ) diff --git a/api/tests/e2e_tests/containers/services/test_dataset_service_update_dataset.py b/api/tests/e2e_tests/containers/services/test_dataset_service_update_dataset.py index fc67f0a4c65c85..350f310461f303 100644 --- a/api/tests/e2e_tests/containers/services/test_dataset_service_update_dataset.py +++ b/api/tests/e2e_tests/containers/services/test_dataset_service_update_dataset.py @@ -411,7 +411,7 @@ def test_update_internal_dataset_indexing_technique_to_high_quality(self, db_ses patch("services.knowledge.dataset_service.current_user", user), patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding" + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding" ) as mock_get_binding, patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task, ): @@ -506,7 +506,7 @@ def test_update_internal_dataset_embedding_model_update(self, db_session_with_co patch("services.knowledge.dataset_service.current_user", user), patch("services.knowledge.dataset_service.ModelManager.for_tenant") as mock_model_manager, patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding" + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding" ) as mock_get_binding, patch("services.knowledge.dataset_service.deal_dataset_vector_index_task") as mock_task, patch("services.knowledge.dataset_service.regenerate_summary_index_task") as mock_regenerate_task, diff --git a/api/tests/e2e_tests/containers/services/test_human_input_delivery_test.py b/api/tests/e2e_tests/containers/services/test_human_input_delivery_test.py index 6dae35ec379b4a..e53a292683fcf8 100644 --- a/api/tests/e2e_tests/containers/services/test_human_input_delivery_test.py +++ b/api/tests/e2e_tests/containers/services/test_human_input_delivery_test.py @@ -7,18 +7,19 @@ import pytest from flask.testing import FlaskClient from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker import services.remote_file_service as remote_file_service_module from enums.human_input import HumanInputFormKind +from extensions.application_services.human_input import build_human_input_debug_service from graphon.enums import BuiltinNodeTypes +from machinery.context import RequestContext from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.human_input import HumanInputForm, HumanInputFormRecipient, HumanInputFormUploadFile from models.human_input_delivery import EmailDeliveryConfig, EmailDeliveryMethod, EmailRecipients, ExternalRecipient from models.human_input_entities import FileInputConfig, HumanInputNodeData from models.model import App, AppMode, UploadFile from models.workflow import Workflow, WorkflowType -from services.workflow_service import WorkflowService def _create_app_with_draft_workflow( @@ -97,6 +98,17 @@ def _create_app_with_draft_workflow( return app, account +def _test_delivery( + *, app_model: App, account: Account, node_id: str, delivery_method_id: str, session: Session +) -> None: + gateway = build_human_input_debug_service(database_client=sessionmaker(session.get_bind(), expire_on_commit=False)) + context = RequestContext("delivery-test", None, account.id, app_model.tenant_id) + app_id = app_model.id + session.expunge_all() + session.rollback() + gateway.test_delivery(context, app_id, node_id, delivery_method_id=delivery_method_id, inputs={}) + + def test_human_input_delivery_test_sends_email( db_session_with_containers, monkeypatch: pytest.MonkeyPatch, @@ -108,8 +120,7 @@ def test_human_input_delivery_test_sends_email( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", send_mock) - service = WorkflowService() - service.test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", @@ -136,7 +147,7 @@ def test_human_input_delivery_test_form_accepts_file_upload( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", MagicMock()) - WorkflowService().test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", @@ -205,7 +216,7 @@ def test_human_input_delivery_test_form_accepts_remote_file_upload( monkeypatch.setattr("services.human_input_delivery_test_service.mail.is_inited", lambda: True) monkeypatch.setattr("services.human_input_delivery_test_service.mail.send", MagicMock()) - WorkflowService().test_human_input_delivery( + _test_delivery( app_model=app, account=account, node_id="human-node", diff --git a/api/tests/e2e_tests/containers/services/test_schedule_service.py b/api/tests/e2e_tests/containers/services/test_schedule_service.py index aeab080a6a2e59..ded5ee13db296e 100644 --- a/api/tests/e2e_tests/containers/services/test_schedule_service.py +++ b/api/tests/e2e_tests/containers/services/test_schedule_service.py @@ -10,11 +10,13 @@ from core.workflow.nodes.trigger_schedule.entities import ScheduleConfig, SchedulePlanUpdate from core.workflow.nodes.trigger_schedule.exc import ScheduleNotFoundError -from events.event_handlers.sync_workflow_schedule_when_app_published import sync_schedule_from_workflow from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.trigger import WorkflowSchedulePlan +from repositories.trigger.workflow_repository import WorkflowTriggerRepository from services.account_errors import AccountNotFoundError from services.trigger.schedule_service import ScheduleService +from services.trigger.workflow_policy import schedule_config +from tests.helpers.unit.model_factories import make_app class ScheduleServiceIntegrationFactory: @@ -117,10 +119,10 @@ def test_create_schedule_persists_schedule(self, db_session_with_containers: Ses with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - schedule = ScheduleService.create_schedule( + schedule = WorkflowTriggerRepository.create_schedule( session=db_session_with_containers, tenant_id=tenant.id, app_id=str(uuid4()), @@ -147,10 +149,10 @@ def test_update_schedule_updates_fields_and_recomputes_next_run(self, db_session with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - updated = ScheduleService.update_schedule( + updated = WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=schedule.id, updates=SchedulePlanUpdate( @@ -180,8 +182,8 @@ def _track[**P](*args: P.args, **kwargs: P.kwargs): calls.append((args, kwargs)) return datetime(2026, 1, 9, 10, 0, 0) - monkeypatch.setattr("services.trigger.schedule_service.calculate_next_run_at", _track) - updated = ScheduleService.update_schedule( + monkeypatch.setattr("repositories.trigger.workflow_repository.calculate_next_run_at", _track) + updated = WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=schedule.id, updates=SchedulePlanUpdate(node_id="node-new"), @@ -194,33 +196,31 @@ def _track[**P](*args: P.args, **kwargs: P.kwargs): def test_update_schedule_not_found_raises(self, db_session_with_containers: Session): with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.update_schedule( + WorkflowTriggerRepository.update_schedule( session=db_session_with_containers, schedule_id=str(uuid4()), updates=SchedulePlanUpdate(node_id="node-new"), ) - def test_delete_schedule_removes_row(self, db_session_with_containers: Session): + def test_sync_schedule_without_config_removes_row(self, db_session_with_containers: Session): _account, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant(db_session_with_containers) schedule = ScheduleServiceIntegrationFactory.create_schedule_plan( - db_session_with_containers, - tenant_id=tenant.id, - ) - - ScheduleService.delete_schedule( - session=db_session_with_containers, - schedule_id=schedule.id, + db_session_with_containers, tenant_id=tenant.id ) + app = make_app(app_id=schedule.app_id, tenant_id=tenant.id) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) db_session_with_containers.commit() - assert db_session_with_containers.get(WorkflowSchedulePlan, schedule.id) is None - def test_delete_schedule_not_found_raises(self, db_session_with_containers: Session): - with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.delete_schedule( - session=db_session_with_containers, - schedule_id=str(uuid4()), - ) + def test_sync_schedule_without_config_is_idempotent_when_missing(self, db_session_with_containers: Session): + app = make_app(app_id=str(uuid4()), tenant_id=str(uuid4())) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) + WorkflowTriggerRepository.sync_schedule(db_session_with_containers, app, None) + db_session_with_containers.commit() + assert ( + db_session_with_containers.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app.id)) + is None + ) def test_get_tenant_owner_returns_owner_account(self, db_session_with_containers: Session): owner, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant( @@ -272,35 +272,6 @@ def test_get_tenant_owner_raises_when_no_owner_or_admin_found(self, db_session_w with pytest.raises(AccountNotFoundError, match=tenant.id): ScheduleService.get_tenant_owner(session=db_session_with_containers, tenant_id=tenant.id) - def test_update_next_run_at_updates_persisted_value(self, db_session_with_containers: Session): - _account, tenant = ScheduleServiceIntegrationFactory.create_account_with_tenant(db_session_with_containers) - schedule = ScheduleServiceIntegrationFactory.create_schedule_plan( - db_session_with_containers, - tenant_id=tenant.id, - ) - expected_next_run = datetime(2026, 1, 3, 10, 30, 0) - - with pytest.MonkeyPatch.context() as monkeypatch: - monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", - lambda *_args, **_kwargs: expected_next_run, - ) - result = ScheduleService.update_next_run_at( - session=db_session_with_containers, - schedule_id=schedule.id, - ) - - db_session_with_containers.refresh(schedule) - assert result == expected_next_run - assert schedule.next_run_at == expected_next_run - - def test_update_next_run_at_raises_when_schedule_not_found(self, db_session_with_containers: Session): - with pytest.raises(ScheduleNotFoundError, match="Schedule not found"): - ScheduleService.update_next_run_at( - session=db_session_with_containers, - schedule_id=str(uuid4()), - ) - class TestSyncScheduleFromWorkflowIntegration: def test_sync_schedule_create_new(self, db_session_with_containers: Session): @@ -310,16 +281,16 @@ def test_sync_schedule_create_new(self, db_session_with_containers: Session): with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_cron_workflow(), + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config(_cron_workflow().graph_dict), ) + db_session_with_containers.commit() - assert result is not None persisted = db_session_with_containers.execute( select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app_id) ).scalar_one() @@ -344,20 +315,22 @@ def test_sync_schedule_update_existing(self, db_session_with_containers: Session with pytest.MonkeyPatch.context() as monkeypatch: monkeypatch.setattr( - "services.trigger.schedule_service.calculate_next_run_at", + "repositories.trigger.workflow_repository.calculate_next_run_at", lambda *_args, **_kwargs: expected_next_run, ) - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_cron_workflow( - node_id="start", - cron_expression="0 12 * * *", - timezone="America/New_York", + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config( + _cron_workflow( + node_id="start", + cron_expression="0 12 * * *", + timezone="America/New_York", + ).graph_dict ), ) + db_session_with_containers.commit() - assert result is not None db_session_with_containers.expire_all() persisted = db_session_with_containers.get(WorkflowSchedulePlan, existing_id) assert persisted is not None @@ -376,12 +349,11 @@ def test_sync_schedule_remove_when_no_config(self, db_session_with_containers: S ) existing_id = existing.id - result = sync_schedule_from_workflow( - tenant_id=tenant.id, - app_id=app_id, - workflow=_no_schedule_workflow(), + WorkflowTriggerRepository.sync_schedule( + db_session_with_containers, + make_app(tenant_id=tenant.id, app_id=app_id), + schedule_config(_no_schedule_workflow().graph_dict), ) - - assert result is None + db_session_with_containers.commit() db_session_with_containers.expire_all() assert db_session_with_containers.get(WorkflowSchedulePlan, existing_id) is None diff --git a/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py b/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py index ad2e9d62fd2f10..1451d72044ad2e 100644 --- a/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py +++ b/api/tests/e2e_tests/containers/services/test_webhook_service_relationships.py @@ -21,6 +21,7 @@ from models.workflow import WorkflowType as ModelWorkflowType from services.errors.app import QuotaExceededError from services.trigger.webhook_service import WebhookService +from services.trigger.workflow_policy import MAX_WEBHOOK_NODES_PER_WORKFLOW class _EndUserServiceStub: @@ -458,7 +459,7 @@ def test_sync_webhook_relationships_raises_when_workflow_exceeds_node_limit( factory = WebhookServiceRelationshipFactory account, tenant = factory.create_account_and_tenant(db_session_with_containers) app = factory.create_app(db_session_with_containers, tenant, account) - node_ids = [f"node-{index}" for index in range(WebhookService.MAX_WEBHOOK_NODES_PER_WORKFLOW + 1)] + node_ids = [f"node-{index}" for index in range(MAX_WEBHOOK_NODES_PER_WORKFLOW + 1)] workflow = factory.create_workflow( db_session_with_containers, app=app, account=account, node_ids=node_ids, version=Workflow.VERSION_DRAFT ) @@ -466,23 +467,6 @@ def test_sync_webhook_relationships_raises_when_workflow_exceeds_node_limit( with pytest.raises(ValueError, match="maximum webhook node limit"): WebhookService.sync_webhook_relationships(app, workflow) - def test_sync_webhook_relationships_raises_when_lock_not_acquired( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=["node-1"], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = False - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - with pytest.raises(RuntimeError, match="Failed to acquire lock"): - WebhookService.sync_webhook_relationships(app, workflow) - def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_records( self, db_session_with_containers: Session, flask_app_with_containers: Flask ): @@ -507,7 +491,8 @@ def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_re ) with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="new-webhook-id-000001" + "repositories.trigger.workflow_repository.WorkflowTriggerRepository.generate_webhook_id", + return_value="new-webhook-id-000001", ): WebhookService.sync_webhook_relationships(app, workflow) @@ -519,66 +504,3 @@ def test_sync_webhook_relationships_creates_missing_records_and_deletes_stale_re assert [record.node_id for record in records] == ["node-new"] assert records[0].webhook_id == "new-webhook-id-000001" assert db_session_with_containers.get(WorkflowWebhookTrigger, stale_trigger_id) is None - - def test_sync_webhook_relationships_sets_redis_cache_for_new_record( - self, db_session_with_containers: Session, flask_app_with_containers: Flask - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, - app=app, - account=account, - node_ids=["node-cache"], - version=Workflow.VERSION_DRAFT, - ) - cache_key = f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:node-cache" - - with patch( - "services.trigger.webhook_service.WebhookService.generate_webhook_id", return_value="cache-webhook-id-00001" - ): - WebhookService.sync_webhook_relationships(app, workflow) - - cached_payload = WebhookServiceRelationshipFactory._read_cache(cache_key) - assert cached_payload is not None - assert cached_payload["node_id"] == "node-cache" - assert cached_payload["webhook_id"] == "cache-webhook-id-00001" - - def test_sync_webhook_relationships_logs_when_lock_release_fails( - self, - db_session_with_containers: Session, - flask_app_with_containers: Flask, - caplog: pytest.LogCaptureFixture, - ): - del flask_app_with_containers - factory = WebhookServiceRelationshipFactory - account, tenant = factory.create_account_and_tenant(db_session_with_containers) - app = factory.create_app(db_session_with_containers, tenant, account) - workflow = factory.create_workflow( - db_session_with_containers, app=app, account=account, node_ids=[], version=Workflow.VERSION_DRAFT - ) - lock = MagicMock() - lock.acquire.return_value = True - lock.release.side_effect = RuntimeError("release failed") - caplog.set_level(logging.ERROR, logger="services.trigger.webhook_service") - - with patch("services.trigger.webhook_service.redis_client.lock", return_value=lock): - WebhookService.sync_webhook_relationships(app, workflow) - - assert caplog.messages.count(f"Failed to release lock for webhook sync, app {app.id}") == 1 - - -def _read_cache(cache_key: str) -> dict[str, str] | None: - from extensions.ext_redis import redis_client - - cached = redis_client.get(cache_key) - if not cached: - return None - if isinstance(cached, bytes): - cached = cached.decode("utf-8") - return json.loads(cached) - - -WebhookServiceRelationshipFactory._read_cache = staticmethod(_read_cache) diff --git a/api/tests/e2e_tests/containers/services/test_workflow_app_service.py b/api/tests/e2e_tests/containers/services/test_workflow_app_service.py index a2c115f6430d52..002c28887ad074 100644 --- a/api/tests/e2e_tests/containers/services/test_workflow_app_service.py +++ b/api/tests/e2e_tests/containers/services/test_workflow_app_service.py @@ -14,8 +14,8 @@ from models.enums import CreatorUserRole, EndUserType from models.workflow import WorkflowAppLogCreatedFrom from models.workflow import WorkflowType as ModelWorkflowType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import WorkflowAppLogQueryService from tests.e2e_tests.containers.helpers import accounts as account_fixtures from tests.e2e_tests.containers.helpers import generate_valid_password @@ -24,7 +24,7 @@ class _WorkflowAppLogTestClient: def __init__(self, session: Session) -> None: session_factory = sessionmaker(bind=session.get_bind(), expire_on_commit=False) self._service = WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=session_factory), + logs=WorkflowAppLogRepository(session_factory=session_factory), ) def get_paginate_workflow_app_logs(self, *, session: Session, app_model, **kwargs): diff --git a/api/tests/e2e_tests/containers/services/test_workflow_draft_variable_service.py b/api/tests/e2e_tests/containers/services/test_workflow_draft_variable_service.py index 21da6f6d6d82e2..8afcde1642606f 100644 --- a/api/tests/e2e_tests/containers/services/test_workflow_draft_variable_service.py +++ b/api/tests/e2e_tests/containers/services/test_workflow_draft_variable_service.py @@ -1,15 +1,16 @@ import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.workflow.variable_prefixes import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID +from extensions.application_services.workflow_variables import build_workflow_variable_service from graphon.variables.segments import StringSegment from models import App, Workflow from models.enums import DraftVariableType from models.workflow import WorkflowDraftVariable -from services.workflow_draft_variable_service import ( +from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository +from services.workflow.draft_variable_service import ( UpdateNotSupportedError, - WorkflowDraftVariableService, ) @@ -17,11 +18,11 @@ def _get_random_variable_name(fake: Faker): return "".join(fake.random_letters(length=10)) -class TestWorkflowDraftVariableService: +class TestWorkflowVariableService: """ - Comprehensive integration tests for WorkflowDraftVariableService using testcontainers. + Comprehensive integration tests for WorkflowVariableService using testcontainers. - This test class covers all major functionality of the WorkflowDraftVariableService: + This test class covers all major functionality of the WorkflowVariableService: - CRUD operations for workflow draft variables (Create, Read, Update, Delete) - Variable listing and filtering by type (conversation, system, node) - Variable updates and resets with proper validation @@ -38,11 +39,11 @@ def mock_external_service_dependencies(self): """ Mock setup for external service dependencies. - WorkflowDraftVariableService doesn't have external dependencies that need mocking, + WorkflowVariableService doesn't have external dependencies that need mocking, so this fixture returns an empty dictionary to maintain consistency with other test classes. This ensures the test structure remains consistent across different service test files. """ - # WorkflowDraftVariableService doesn't have external dependencies that need mocking + # WorkflowVariableService doesn't have external dependencies that need mocking return {} def _create_test_app( @@ -207,8 +208,10 @@ def test_get_variable_success(self, db_session_with_containers: Session, mock_ex user_id=app.created_by, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variable = service.get_variable(variable.id) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + retrieved_variable = service.get_variable(variable.id, app_id=app.id, user_id=app.created_by) assert retrieved_variable is not None assert retrieved_variable.id == variable.id assert retrieved_variable.name == "test_var" @@ -225,8 +228,14 @@ def test_get_variable_not_found(self, db_session_with_containers: Session, mock_ """ fake = Faker() non_existent_id = fake.uuid4() - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variable = service.get_variable(non_existent_id) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + retrieved_variable = service.get_variable( + non_existent_id, + app_id=fake.uuid4(), + user_id=fake.uuid4(), + ) assert retrieved_variable is None def test_get_draft_variables_by_selectors_success( @@ -265,8 +274,9 @@ def test_get_draft_variables_by_selectors_success( [CONVERSATION_VARIABLE_NODE_ID, "var2"], ["test_node_1", "var3"], ] - service = WorkflowDraftVariableService(db_session_with_containers) - retrieved_variables = service.get_draft_variables_by_selectors(app.id, selectors, user_id=app.created_by) + retrieved_variables = WorkflowDraftVariableRepository( + sessions=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ).get_draft_variables_by_selectors(app.id, selectors, user_id=app.created_by) assert len(retrieved_variables) == 3 var_names = [var.name for var in retrieved_variables] assert "var1" in var_names @@ -303,7 +313,9 @@ def test_list_variables_without_values_success( test_value, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_variables_without_values(app.id, page=1, limit=3, user_id=app.created_by) assert result.total == 5 assert len(result.variables) == 3 @@ -354,7 +366,9 @@ def test_list_node_variables_success(self, db_session_with_containers: Session, variable_type=DraftVariableType.NODE, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_node_variables(app.id, node_id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -396,7 +410,9 @@ def test_list_conversation_variables_success( variable_type=DraftVariableType.SYS, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_conversation_variables(app.id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -428,16 +444,20 @@ def test_update_variable_success(self, db_session_with_containers: Session, mock original_value, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) updated_variable = service.update_variable(variable, name="new_name", value=new_value) assert updated_variable.name == "new_name" assert updated_variable.get_value().value == new_value.value assert updated_variable.last_edited_at is not None - db_session_with_containers.refresh(variable) - assert variable.name == "new_name" - assert variable.get_value().value == new_value.value - assert variable.last_edited_at is not None + with Session(db_session_with_containers.get_bind()) as verify: + persisted = verify.get(WorkflowDraftVariable, variable.id) + assert persisted is not None + assert persisted.name == "new_name" + assert persisted.get_value().value == new_value.value + assert persisted.last_edited_at is not None def test_update_variable_not_editable( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -464,7 +484,9 @@ def test_update_variable_not_editable( db_session_with_containers.add(variable) db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) with pytest.raises(UpdateNotSupportedError) as exc_info: service.update_variable(variable, name="new_name", value=new_value) assert "variable not support updating" in str(exc_info.value) @@ -506,14 +528,22 @@ def test_reset_conversation_variable_success( ) variable.last_edited_at = fake.date_time() db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + db_session_with_containers.refresh(workflow) + db_session_with_containers.refresh(variable) + db_session_with_containers.expunge(workflow) + db_session_with_containers.expunge(variable) + db_session_with_containers.rollback() + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) reset_variable = service.reset_variable(workflow, variable) assert reset_variable is not None assert reset_variable.get_value().value == "default_value" assert reset_variable.last_edited_at is None - db_session_with_containers.refresh(variable) - assert variable.get_value().value == "default_value" - assert variable.last_edited_at is None + persisted = db_session_with_containers.get(WorkflowDraftVariable, variable.id) + assert persisted is not None + assert persisted.get_value().value == "default_value" + assert persisted.last_edited_at is None def test_delete_variable_success(self, db_session_with_containers: Session, mock_external_service_dependencies): """ @@ -531,7 +561,11 @@ def test_delete_variable_success(self, db_session_with_containers: Session, mock ) assert db_session_with_containers.query(WorkflowDraftVariable).filter_by(id=variable.id).first() is not None - service = WorkflowDraftVariableService(db_session_with_containers) + db_session_with_containers.expunge(variable) + db_session_with_containers.rollback() + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_variable(variable) assert db_session_with_containers.query(WorkflowDraftVariable).filter_by(id=variable.id).first() is None @@ -574,7 +608,9 @@ def test_delete_workflow_variables_success( ) assert len(app_variables) == 3 assert len(other_app_variables) == 1 - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_user_workflow_variables(app.id, user_id=app.created_by) app_variables_after = db_session_with_containers.query(WorkflowDraftVariable).filter_by(app_id=app.id).all() other_app_variables_after = ( @@ -628,7 +664,9 @@ def test_draft_variables_are_isolated_between_users( fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) user_a_vars = service.list_conversation_variables(app.id, user_id=user_a) user_b_vars = service.list_conversation_variables(app.id, user_id=user_b) @@ -705,7 +743,9 @@ def test_delete_node_variables_success( assert len(target_node_variables) == 2 assert len(other_node_variables) == 1 assert len(conv_variables) == 1 - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.delete_node_variables(app.id, node_id, user_id=app.created_by) target_node_variables_after = ( db_session_with_containers.query(WorkflowDraftVariable).filter_by(app_id=app.id, node_id=node_id).all() @@ -753,7 +793,9 @@ def test_prefill_conversation_variable_default_values_success( workflow.conversation_variables = [conv_var1, conv_var2] db_session_with_containers.commit() - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) service.prefill_conversation_variable_default_values(workflow, user_id="00000000-0000-0000-0000-000000000001") draft_variables = ( db_session_with_containers.query(WorkflowDraftVariable) @@ -793,7 +835,9 @@ def test_get_conversation_id_from_draft_variable_success( variable_type=DraftVariableType.SYS, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_id = service._get_conversation_id_from_draft_variable(app.id, app.created_by) assert retrieved_conv_id == conversation_id @@ -809,7 +853,9 @@ def test_get_conversation_id_from_draft_variable_not_found( """ fake = Faker() app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, fake=fake) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_id = service._get_conversation_id_from_draft_variable(app.id, app.created_by) assert retrieved_conv_id is None @@ -850,7 +896,9 @@ def test_list_system_variables_success( self._create_test_variable( db_session_with_containers, app.id, CONVERSATION_VARIABLE_NODE_ID, "conv_var", conv_var_value, fake=fake ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) result = service.list_system_variables(app.id, user_id=app.created_by) assert len(result.variables) == 2 for var in result.variables: @@ -897,7 +945,9 @@ def test_get_variable_by_name_success( variable_type=DraftVariableType.NODE, fake=fake, ) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_var = service.get_conversation_variable(app.id, "test_conv_var", user_id=app.created_by) assert retrieved_conv_var is not None assert retrieved_conv_var.name == "test_conv_var" @@ -923,7 +973,9 @@ def test_get_variable_by_name_not_found( """ fake = Faker() app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, fake=fake) - service = WorkflowDraftVariableService(db_session_with_containers) + service = build_workflow_variable_service( + database_client=sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) retrieved_conv_var = service.get_conversation_variable(app.id, "non_existent_conv_var", user_id=app.created_by) assert retrieved_conv_var is None retrieved_sys_var = service.get_system_variable(app.id, "non_existent_sys_var", user_id=app.created_by) diff --git a/api/tests/e2e_tests/containers/services/test_workflow_run_service.py b/api/tests/e2e_tests/containers/services/test_workflow_run_service.py index b522e713116986..70787a263dc1e3 100644 --- a/api/tests/e2e_tests/containers/services/test_workflow_run_service.py +++ b/api/tests/e2e_tests/containers/services/test_workflow_run_service.py @@ -20,6 +20,7 @@ from models.workflow import WorkflowType as ModelWorkflowType from repositories.factory import DifyAPIRepositoryFactory from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from repositories.tools.provider_repository import ToolProviderRepository from services.app_service import AppService, CreateAppParams from services.workflow_run_service import WorkflowRunService from tests.e2e_tests.containers.helpers import accounts as account_fixtures @@ -41,6 +42,7 @@ def workflow_run_service(self, db_session_with_containers: Session) -> WorkflowR node_executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( session_maker=session_factory ), + tool_providers=ToolProviderRepository(session_factory), ) @pytest.fixture diff --git a/api/tests/e2e_tests/containers/services/test_workflow_service.py b/api/tests/e2e_tests/containers/services/test_workflow_service.py index 0ff2a85d4cb9c4..9367118b0bc046 100644 --- a/api/tests/e2e_tests/containers/services/test_workflow_service.py +++ b/api/tests/e2e_tests/containers/services/test_workflow_service.py @@ -12,11 +12,15 @@ from faker import Faker from sqlalchemy.orm import Session +from extensions.application_services.workflow import build_app_dsl_service from graphon.enums import BuiltinNodeTypes, ErrorStrategy, WorkflowNodeExecutionStatus +from machinery.context import RequestContext from models import Account, AccountStatus, App, TenantStatus, Workflow from models.enums import CreatorUserRole from models.model import AppMode from models.workflow import WorkflowType +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.workflow.contracts import DraftSyncCommand, WorkflowOwner from services.workflow_ref_service import WorkflowRef from services.workflow_service import WorkflowService from tests.helpers.unit.core.model_fixtures import make_model_instance @@ -461,10 +465,8 @@ def test_get_all_published_workflow_pagination(self, db_session_with_containers: db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - First page - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -477,7 +479,7 @@ def test_get_all_published_workflow_pagination(self, db_session_with_containers: assert has_more is True # Act - Second page - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=2, @@ -517,10 +519,8 @@ def test_get_all_published_workflow_user_filter(self, db_session_with_containers db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - Filter by account1 - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=account1.id ) @@ -559,10 +559,8 @@ def test_get_all_published_workflow_named_only_filter(self, db_session_with_cont db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - Filter named only - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None, named_only=True ) @@ -578,10 +576,8 @@ def test_get_all_published_workflow_no_workflow_id(self, db_session_with_contain app.workflow_id = None db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None ) @@ -604,10 +600,8 @@ def test_get_all_published_workflow_basic(self, db_session_with_containers: Sess app.workflow_id = workflow1.id db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, limit=10, user_id=None ) @@ -644,10 +638,8 @@ def test_get_all_published_workflow_combined_filters(self, db_session_with_conta app.workflow_id = wf1.id db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - Filter by account1 + named_only - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -673,10 +665,8 @@ def test_get_all_published_workflow_empty_result(self, db_session_with_container app.workflow_id = workflow.id db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - Filter by a user that has no workflows - result_workflows, has_more = workflow_service.get_all_published_workflow( + result_workflows, has_more = WorkflowDefinitionStore.get_all_published_workflow( session=db_session_with_containers, app_model=app, page=1, @@ -717,18 +707,22 @@ def test_sync_draft_workflow_create_new(self, db_session_with_containers: Sessio environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=unique_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, + result = drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + WorkflowOwner(app.id), + DraftSyncCommand( + graph=graph, + features=features, + unique_hash=unique_hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) # Assert @@ -736,7 +730,7 @@ def test_sync_draft_workflow_create_new(self, db_session_with_containers: Sessio assert result.version == Workflow.VERSION_DRAFT assert result.app_id == app.id assert result.tenant_id == app.tenant_id - assert result.unique_hash is not None # Should have a hash generated + assert result.hash is not None # Should have a hash generated assert result.graph == json.dumps(graph) assert result.features == json.dumps(features) assert result.created_by == account.id @@ -782,18 +776,22 @@ def test_sync_draft_workflow_update_existing(self, db_session_with_containers: S environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=new_graph, - features=new_features, - unique_hash=original_hash, # Use original hash to allow update - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, + result = drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + WorkflowOwner(app.id), + DraftSyncCommand( + graph=new_graph, + features=new_features, + unique_hash=original_hash, # Use original hash to allow update + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) # Assert @@ -801,7 +799,7 @@ def test_sync_draft_workflow_update_existing(self, db_session_with_containers: S assert result.id == existing_workflow.id # Same workflow updated assert result.version == Workflow.VERSION_DRAFT # Hash should be updated to reflect new content - assert result.unique_hash != original_hash # Hash should change after update + assert result.hash != original_hash # Hash should change after update assert result.graph == json.dumps(new_graph) assert result.features == json.dumps(new_features) assert result.updated_by == account.id @@ -839,177 +837,27 @@ def test_sync_draft_workflow_hash_mismatch_error(self, db_session_with_container environment_variables = [] conversation_variables = [] - workflow_service = WorkflowService() + drafts = build_app_dsl_service(db_session_with_containers)._drafts # Act & Assert from services.errors.app import WorkflowHashNotEqualError with pytest.raises(WorkflowHashNotEqualError): - workflow_service.sync_draft_workflow( - app_model=app, - graph=new_graph, - features=new_features, - unique_hash=mismatched_hash, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=db_session_with_containers, - ) - - def test_publish_workflow_success(self, db_session_with_containers: Session): - """ - Test successful workflow publishing. - - This test verifies that the service can correctly publish a draft - workflow, creating a new published version with proper versioning - and status management. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Create draft workflow - workflow = self._create_test_workflow(db_session_with_containers, app, account, fake) - workflow.version = Workflow.VERSION_DRAFT - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - # Act - Mock current_user context and pass session - from unittest.mock import patch - - with patch("flask_login.utils._get_user", return_value=account, autospec=True): - result = workflow_service.publish_workflow( - session=db_session_with_containers, app_model=app, account=account + drafts.sync( + RequestContext("test", None, account.id, app.tenant_id), + WorkflowOwner(app.id), + DraftSyncCommand( + graph=new_graph, + features=new_features, + unique_hash=mismatched_hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=environment_variables, + conversation_variables=conversation_variables, + ), ) - # Assert - assert result is not None - assert result.version != Workflow.VERSION_DRAFT - # Version should be a timestamp format like '2025-08-22 00:10:24.722051' - assert isinstance(result.version, str) - assert len(result.version) > 10 # Should be a reasonable timestamp length - assert result.created_by == account.id - - def test_publish_workflow_no_draft_error(self, db_session_with_containers: Session): - """ - Test error when publishing workflow without draft. - - This test ensures that the service correctly prevents publishing - when no draft workflow exists, maintaining workflow state consistency. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Don't create any workflow - app should have no draft - - workflow_service = WorkflowService() - - # Act & Assert - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app, account=account) - - def test_publish_workflow_already_published_error(self, db_session_with_containers: Session): - """ - Test error when publishing already published workflow. - - This test ensures that the service correctly prevents re-publishing - of already published workflows, maintaining version control integrity. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - - # Create already published workflow - workflow = self._create_test_workflow(db_session_with_containers, app, account, fake) - workflow.version = "2024.01.01.001" # Already published - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - # Act & Assert - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app, account=account) - - def test_restore_published_workflow_to_draft_does_not_persist_normalized_source_features( - self, db_session_with_containers: Session - ): - """Restore copies legacy feature JSON into draft without rewriting the source row.""" - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.ADVANCED_CHAT - - legacy_features = { - "file_upload": { - "image": { - "enabled": True, - "number_limits": 6, - "transfer_methods": ["remote_url", "local_file"], - } - }, - "opening_statement": "", - "retriever_resource": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": False}, - "speech_to_text": {"enabled": False}, - "suggested_questions": [], - "suggested_questions_after_answer": {"enabled": False}, - "text_to_speech": {"enabled": False, "language": "", "voice": ""}, - } - published_workflow = Workflow( - id=fake.uuid4(), - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version="2026.03.19.001", - graph=json.dumps({"nodes": [], "edges": []}), - _features=json.dumps(legacy_features), - created_by=account.id, - updated_by=account.id, - _environment_variables=[], - _conversation_variables=[], - ) - draft_workflow = Workflow( - id=fake.uuid4(), - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps({"nodes": [], "edges": []}), - _features=json.dumps({}), - created_by=account.id, - updated_by=account.id, - _environment_variables=[], - _conversation_variables=[], - ) - db_session_with_containers.add(published_workflow) - db_session_with_containers.add(draft_workflow) - db_session_with_containers.commit() - - workflow_service = WorkflowService() - - restored_workflow = workflow_service.restore_published_workflow_to_draft( - app_model=app, workflow_id=published_workflow.id, account=account, session=db_session_with_containers - ) - - db_session_with_containers.expire_all() - refreshed_published_workflow = ( - db_session_with_containers.query(Workflow).filter_by(id=published_workflow.id).first() - ) - refreshed_draft_workflow = db_session_with_containers.query(Workflow).filter_by(id=draft_workflow.id).first() - - assert restored_workflow.id == draft_workflow.id - assert refreshed_published_workflow is not None - assert refreshed_draft_workflow is not None - assert refreshed_published_workflow.serialized_features == json.dumps(legacy_features) - assert refreshed_draft_workflow.serialized_features == json.dumps(legacy_features) - def test_get_default_block_configs(self, db_session_with_containers: Session): """ Test retrieval of default block configurations for all node types. @@ -1094,156 +942,6 @@ def test_get_default_block_config_with_filters(self, db_session_with_containers: # Result might be None if filters don't match, but should not raise error assert result is None or isinstance(result, dict) - def test_convert_to_workflow_chat_mode_success(self, db_session_with_containers: Session): - """ - Test successful conversion from chat mode app to workflow mode. - - This test verifies that the service can correctly convert a chatbot - application to workflow mode, which is essential for app mode migration. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create chat mode app - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.CHAT - - # Create app model config (required for conversion) - from models.model import AppModelConfig - - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - # Set the model field directly - this is what model_dict property returns - model=json.dumps( - { - "provider": "openai", - "name": "gpt-3.5-turbo", - "completion_params": {"max_tokens": 1000, "temperature": 0.7}, - } - ), - # Set pre_prompt for PromptTemplateConfigManager - pre_prompt="You are a helpful assistant.", - created_by=account.id, - updated_by=account.id, - ) - app_model_config.id = fake.uuid4() - - db_session_with_containers.add(app_model_config) - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = { - "name": "Converted Workflow App", - "icon_type": "emoji", - "icon": "🚀", - "icon_background": "#FF5733", - } - - # Act - result = workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - - # Assert - assert result is not None - assert result.mode == AppMode.ADVANCED_CHAT # CHAT mode converts to ADVANCED_CHAT, not WORKFLOW - assert result.name == conversion_args["name"] - assert result.icon == conversion_args["icon"] - assert result.icon_type == conversion_args["icon_type"] - assert result.icon_background == conversion_args["icon_background"] - - def test_convert_to_workflow_completion_mode_success(self, db_session_with_containers: Session): - """ - Test successful conversion from completion mode app to workflow mode. - - This test verifies that the service can correctly convert a completion - application to workflow mode, supporting different app type migrations. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create completion mode app - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.COMPLETION - - # Create app model config (required for conversion) - from models.model import AppModelConfig - - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model_id="gpt-3.5-turbo", - # Set the model field directly - this is what model_dict property returns - model=json.dumps( - { - "provider": "openai", - "name": "gpt-3.5-turbo", - "completion_params": {"max_tokens": 1000, "temperature": 0.7}, - } - ), - # Set pre_prompt for PromptTemplateConfigManager - pre_prompt="Complete the following text:", - created_by=account.id, - updated_by=account.id, - ) - app_model_config.id = fake.uuid4() - - db_session_with_containers.add(app_model_config) - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = { - "name": "Converted Workflow App", - "icon_type": "emoji", - "icon": "🚀", - "icon_background": "#FF5733", - } - - # Act - result = workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - - # Assert - assert result is not None - assert result.mode == AppMode.WORKFLOW - assert result.name == conversion_args["name"] - assert result.icon == conversion_args["icon"] - assert result.icon_type == conversion_args["icon_type"] - assert result.icon_background == conversion_args["icon_background"] - - def test_convert_to_workflow_unsupported_mode_error(self, db_session_with_containers: Session): - """ - Test error when attempting to convert unsupported app mode. - - This test ensures that the service correctly prevents conversion - of apps that are not in supported modes for workflow conversion. - """ - # Arrange - fake = Faker() - account = self._create_test_account(db_session_with_containers, fake) - - # Create workflow mode app (already in workflow mode) - app = self._create_test_app(db_session_with_containers, fake) - app.mode = AppMode.WORKFLOW - - db_session_with_containers.commit() - - workflow_service = WorkflowService() - conversion_args = {"name": "Test"} - - # Act & Assert - with pytest.raises(ValueError, match="Current App mode: workflow is not supported convert to workflow"): - workflow_service.convert_to_workflow( - app_model=app, account=account, args=conversion_args, session=db_session_with_containers - ) - def test_validate_features_structure_advanced_chat(self, db_session_with_containers: Session): """ Test feature structure validation for advanced chat mode apps. @@ -1334,11 +1032,10 @@ def test_update_workflow_success(self, db_session_with_containers: Session): db_session_with_containers.commit() - workflow_service = WorkflowService() update_data = {"marked_name": "Updated Workflow Name", "marked_comment": "Updated workflow comment"} # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1363,12 +1060,11 @@ def test_update_workflow_not_found(self, db_session_with_containers: Session): account = self._create_test_account(db_session_with_containers, fake) app = self._create_test_app(db_session_with_containers, fake) - workflow_service = WorkflowService() non_existent_workflow_id = fake.uuid4() update_data = {"marked_name": "Test"} # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1393,7 +1089,6 @@ def test_update_workflow_ignores_disallowed_fields(self, db_session_with_contain db_session_with_containers.commit() - workflow_service = WorkflowService() update_data = { "marked_name": "Allowed Update", "graph": "disallowed_field", # Should be ignored @@ -1401,7 +1096,7 @@ def test_update_workflow_ignores_disallowed_fields(self, db_session_with_contain } # Act - result = workflow_service.update_workflow( + result = WorkflowDefinitionStore.update_workflow( session=db_session_with_containers, account_id=account.id, data=update_data, @@ -1434,10 +1129,8 @@ def test_delete_workflow_success(self, db_session_with_containers: Session): db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act - result = workflow_service.delete_workflow( + result = WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1467,13 +1160,11 @@ def test_delete_workflow_draft_error(self, db_session_with_containers: Session): db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act & Assert from services.errors.workflow_service import DraftWorkflowDeletionError with pytest.raises(DraftWorkflowDeletionError, match="Cannot delete draft workflow versions"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1499,13 +1190,11 @@ def test_delete_workflow_in_use_error(self, db_session_with_containers: Session) db_session_with_containers.commit() - workflow_service = WorkflowService() - # Act & Assert from services.errors.workflow_service import WorkflowInUseError with pytest.raises(WorkflowInUseError, match="Cannot delete workflow that is currently in use by app"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=workflow.tenant_id, owner_id=app.id, workflow_id=workflow.id), ) @@ -1522,11 +1211,9 @@ def test_delete_workflow_not_found_error(self, db_session_with_containers: Sessi app = self._create_test_app(db_session_with_containers, fake) non_existent_workflow_id = fake.uuid4() - workflow_service = WorkflowService() - # Act & Assert with pytest.raises(ValueError, match=f"Workflow with ID {non_existent_workflow_id} not found"): - workflow_service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef( tenant_id=app.tenant_id, @@ -1567,7 +1254,7 @@ def test_run_free_workflow_node_success(self, db_session_with_containers: Sessio from unittest.mock import patch - from core.workflow.node_factory import DifyNodeFactory + from services.workflow.execution.adapters.node_factory import DifyNodeFactory # Act with patch.object( diff --git a/api/tests/e2e_tests/containers/services/tools/conftest.py b/api/tests/e2e_tests/containers/services/tools/conftest.py new file mode 100644 index 00000000000000..187b5649fe7d00 --- /dev/null +++ b/api/tests/e2e_tests/containers/services/tools/conftest.py @@ -0,0 +1,17 @@ +"""Tool metadata and management use the test database through explicit ports.""" + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from repositories.tools.workflow_repository import WorkflowToolRepository +from services.tools.workflow_tools_manage_service import WorkflowToolManageService + + +@pytest.fixture +def workflow_queries(db_session_with_containers: Session) -> WorkflowToolRepository: + return WorkflowToolRepository(sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False)) + + +@pytest.fixture +def workflow_tools(workflow_queries: WorkflowToolRepository) -> WorkflowToolManageService: + return WorkflowToolManageService(workflow_queries) diff --git a/api/tests/e2e_tests/containers/services/tools/test_mcp_tools_manage_service.py b/api/tests/e2e_tests/containers/services/tools/test_mcp_tools_manage_service.py index 73a7b1002866f4..6ee825442c7b35 100644 --- a/api/tests/e2e_tests/containers/services/tools/test_mcp_tools_manage_service.py +++ b/api/tests/e2e_tests/containers/services/tools/test_mcp_tools_manage_service.py @@ -2,12 +2,13 @@ import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from core.tools.entities.tool_entities import ToolProviderType from models import Account, Tenant from models.tools import MCPToolProvider -from services.tools.legacy_mcp_tools_manage_service import UNCHANGED_SERVER_URL_PLACEHOLDER, MCPToolManageService +from repositories.tools.provider_repository import ToolProviderRepository +from services.tools.mcp_tools_manage_service import UNCHANGED_SERVER_URL_PLACEHOLDER, MCPToolManageService class TestMCPToolManageService: @@ -17,8 +18,8 @@ class TestMCPToolManageService: def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" with ( - patch("services.tools.legacy_mcp_tools_manage_service.encrypter") as mock_encrypter, - patch("services.tools.legacy_mcp_tools_manage_service.ToolTransformService") as mock_tool_transform_service, + patch("services.tools.mcp_tools_manage_service.encrypter") as mock_encrypter, + patch("services.tools.mcp_tools_manage_service.ToolTransformService") as mock_tool_transform_service, ): # Setup default mock returns from core.tools.entities.api_entities import ToolProviderApiEntity @@ -378,9 +379,7 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, ) # Assert: Verify the expected outcomes - assert result is not None - assert result.name == "Test MCP Provider" - assert result.type == ToolProviderType.MCP + assert isinstance(result, str) # Verify database state @@ -391,6 +390,8 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, ) assert created_provider is not None + assert result == created_provider.id + assert created_provider.name == "Test MCP Provider" assert created_provider.server_identifier == "test_identifier_123" assert created_provider.timeout == 30.0 assert created_provider.sse_read_timeout == 300.0 @@ -401,7 +402,7 @@ def test_create_mcp_provider_success(self, db_session_with_containers: Session, mock_external_service_dependencies["encrypter"].encrypt_token.assert_called_once_with( tenant.id, "https://example.com/mcp" ) - mock_external_service_dependencies["tool_transform_service"].mcp_provider_to_user_provider.assert_called_once() + mock_external_service_dependencies["tool_transform_service"].mcp_provider_to_user_provider.assert_not_called() def test_create_mcp_provider_duplicate_name( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -638,8 +639,10 @@ def test_retrieve_mcp_tools_success(self, db_session_with_containers: Session, m # Act: Execute the method under test - service = MCPToolManageService(db_session_with_containers) - result = service.list_providers(tenant_id=tenant.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result = MCPToolManageService.list_providers(tenant_id=tenant.id, tool_providers=tool_providers) # Assert: Verify the expected outcomes assert result is not None @@ -676,8 +679,10 @@ def test_retrieve_mcp_tools_empty_list( # Act: Execute the method under test - service = MCPToolManageService(db_session_with_containers) - result = service.list_providers(tenant_id=tenant.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result = MCPToolManageService.list_providers(tenant_id=tenant.id, tool_providers=tool_providers) # Assert: Verify the expected outcomes assert result is not None @@ -749,9 +754,11 @@ def test_retrieve_mcp_tools_tenant_isolation( # Act: Execute the method under test for both tenants - service = MCPToolManageService(db_session_with_containers) - result1 = service.list_providers(tenant_id=tenant1.id) - result2 = service.list_providers(tenant_id=tenant2.id) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + result1 = MCPToolManageService.list_providers(tenant_id=tenant1.id, tool_providers=tool_providers) + result2 = MCPToolManageService.list_providers(tenant_id=tenant2.id, tool_providers=tool_providers) # Assert: Verify tenant isolation assert len(result1) == 1 @@ -804,7 +811,7 @@ def test_list_mcp_tool_from_remote_server_success( )(), ] - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: # Setup mock client mock_client_instance = mock_mcp_client.return_value.__enter__.return_value mock_client_instance.list_tools.return_value = mock_tools @@ -866,7 +873,7 @@ def test_list_mcp_tool_from_remote_server_auth_error( mock_decrypt.return_value = "https://example.com/mcp" # Mock MCPClient to raise authentication error - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: from core.mcp.error import MCPAuthError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value @@ -918,7 +925,7 @@ def test_list_mcp_tool_from_remote_server_connection_error( mock_decrypt.return_value = "https://example.com/mcp" # Mock MCPClient to raise connection error - with patch("services.tools.legacy_mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: + with patch("services.tools.mcp_tools_manage_service.MCPClientWithAuthRetry") as mock_mcp_client: from core.mcp.error import MCPError mock_client_instance = mock_mcp_client.return_value.__enter__.return_value diff --git a/api/tests/e2e_tests/containers/services/tools/test_tools_transform_service.py b/api/tests/e2e_tests/containers/services/tools/test_tools_transform_service.py index ef7c7c7130f0b5..97c179480d4be2 100644 --- a/api/tests/e2e_tests/containers/services/tools/test_tools_transform_service.py +++ b/api/tests/e2e_tests/containers/services/tools/test_tools_transform_service.py @@ -1,11 +1,12 @@ from __future__ import annotations -from unittest.mock import Mock, patch +from unittest.mock import Mock, create_autospec, patch import pytest from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.plugin.plugin_service import PluginService from core.tools.entities.api_entities import ToolApiEntity, ToolProviderApiEntity from core.tools.entities.common_entities import I18nObject @@ -17,7 +18,8 @@ ToolProviderType, ) from models.tools import ApiToolProvider, WorkflowToolProvider -from services.tools.legacy_tools_transform_service import ToolTransformService +from repositories.tools.provider_repository import ToolProviderRepository +from services.tools.tools_transform_service import ToolTransformService from tests.tool_fixtures import make_runtime_tool @@ -27,7 +29,7 @@ class TestToolTransformService: @pytest.fixture def mock_external_service_dependencies(self): """Mock setup for external service dependencies.""" - with patch("services.tools.legacy_tools_transform_service.dify_config") as mock_dify_config: + with patch("services.tools.tools_transform_service.dify_config") as mock_dify_config: with patch("core.plugin.plugin_service.dify_config", new=mock_dify_config): # Setup default mock returns mock_dify_config.CONSOLE_API_URL = "https://console.example.com" @@ -462,7 +464,7 @@ def test_builtin_provider_to_user_provider_success( mock_db_provider.credentials = {"api_key": "encrypted_key"} # Mock encryption - with patch("services.tools.legacy_tools_transform_service.create_provider_encrypter") as mock_encrypter: + with patch("services.tools.tools_transform_service.create_provider_encrypter") as mock_encrypter: mock_encrypter_instance = Mock() mock_encrypter_instance.decrypt.return_value = {"api_key": "decrypted_key"} mock_encrypter_instance.mask_plugin_credentials.return_value = {"api_key": ""} @@ -607,12 +609,31 @@ def test_api_provider_to_controller_success( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") - # Additional assertions would depend on the actual controller implementation + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.entity.identity.icon == provider_record.icon + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key_header", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == { + "auth_type", + "api_key_header", + "api_key_value", + "api_key_header_prefix", + } + assert credentials_schema["api_key_header"].default == "Authorization" + assert credentials_schema["api_key_header_prefix"].default == "basic" def test_api_provider_to_controller_api_key_query( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -643,11 +664,24 @@ def test_api_provider_to_controller_api_key_query( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key_query", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == {"auth_type", "api_key_query_param", "api_key_value"} + assert credentials_schema["api_key_query_param"].default == "key" def test_api_provider_to_controller_backward_compatibility( self, db_session_with_containers: Session, mock_external_service_dependencies @@ -679,14 +713,31 @@ def test_api_provider_to_controller_backward_compatibility( db_session_with_containers.commit() # Act: Execute the method under test - result = ToolTransformService.api_provider_to_controller(provider) + tool_providers = ToolProviderRepository( + sessionmaker(bind=db_session_with_containers.get_bind(), expire_on_commit=False) + ) + provider_record = tool_providers.get(tenant_id=provider.tenant_id, provider_id=provider.id) + assert provider_record is not None + result = ToolTransformService.api_provider_to_controller(provider_record) # Assert: Verify the expected outcomes - assert result is not None - assert hasattr(result, "from_db") + assert result.provider_id == provider_record.id + assert result.tenant_id == provider_record.tenant_id + assert result.provider_type is ToolProviderType.API + assert result.entity.identity.name == provider_record.name + assert result.entity.identity.author == provider_record.author + assert result.tools == [] + assert provider_record.credentials == {"auth_type": "api_key", "api_key": "test_key"} + credentials_schema = {config.name: config for config in result.entity.credentials_schema} + assert set(credentials_schema) == { + "auth_type", + "api_key_header", + "api_key_value", + "api_key_header_prefix", + } def test_workflow_provider_to_controller_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_queries ): """ Test successful conversion of workflow provider to controller. @@ -715,19 +766,22 @@ def test_workflow_provider_to_controller_success( db_session_with_containers.commit() # Mock the WorkflowToolProviderController.from_db method to avoid app dependency - with patch( - "services.tools.legacy_tools_transform_service.WorkflowToolProviderController.from_db" - ) as mock_from_db: + with patch("services.tools.tools_transform_service.WorkflowToolProviderController.from_db") as mock_from_db: mock_controller = Mock() mock_from_db.return_value = mock_controller # Act: Execute the method under test - result = ToolTransformService.workflow_provider_to_controller(provider) + result = ToolTransformService.workflow_provider_to_controller(provider, queries=workflow_queries) # Assert: Verify the expected outcomes assert result is not None assert result == mock_controller - mock_from_db.assert_called_once_with(provider) + mock_from_db.assert_called_once_with( + provider, + queries=workflow_queries, + draft_variable_saver=None, + workflow_runtime=None, + ) def _param(name, *, form=ToolParameter.ToolParameterForm.FORM, label=None): @@ -811,8 +865,8 @@ class TestWorkflowProviderToUserProvider: """Tests for ToolTransformService.workflow_provider_to_user_provider.""" @staticmethod - def _make_controller(provider_id="provider_123", **identity_overrides): - from core.tools.workflow_as_tool.provider import WorkflowToolProviderController + def _make_controller(provider_id="provider_123", *, workflow_queries, **identity_overrides): + from services.tools.workflow.provider import WorkflowToolProviderController defaults = { "author": "test_author", @@ -825,10 +879,17 @@ def _make_controller(provider_id="provider_123", **identity_overrides): defaults.update(identity_overrides) identity = ToolProviderIdentity(**defaults) entity = ToolProviderEntity(identity=identity) - return WorkflowToolProviderController(entity=entity, provider_id=provider_id) + return WorkflowToolProviderController( + draft_variable_saver=Mock( + return_value=create_autospec(DraftVariableSaverFactory, instance=True, spec_set=True) + ), + entity=entity, + provider_id=provider_id, + queries=workflow_queries, + ) - def test_with_workflow_app_id(self): - ctrl = self._make_controller() + def test_with_workflow_app_id(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -843,8 +904,8 @@ def test_with_workflow_app_id(self): assert result.labels == ["l1", "l2"] assert result.is_team_authorization is True - def test_without_workflow_app_id(self): - ctrl = self._make_controller() + def test_without_workflow_app_id(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -853,8 +914,8 @@ def test_without_workflow_app_id(self): assert result.workflow_app_id is None - def test_workflow_app_id_none_explicit(self): - ctrl = self._make_controller() + def test_workflow_app_id_none_explicit(self, *, workflow_queries): + ctrl = self._make_controller(workflow_queries=workflow_queries) result = ToolTransformService.workflow_provider_to_user_provider( provider_controller=ctrl, @@ -865,7 +926,7 @@ def test_workflow_app_id_none_explicit(self): assert result.workflow_app_id is None assert result.labels == [] - def test_preserves_other_fields(self): + def test_preserves_other_fields(self, *, workflow_queries): ctrl = self._make_controller( "provider_456", author="another_author", @@ -874,6 +935,7 @@ def test_preserves_other_fields(self): icon='{"type": "emoji", "content": "⚙️"}', icon_dark='{"type": "emoji", "content": "🔧"}', label=I18nObject(en_US="Another Tool", zh_Hans="Another Tool"), + workflow_queries=workflow_queries, ) result = ToolTransformService.workflow_provider_to_user_provider( diff --git a/api/tests/e2e_tests/containers/services/tools/test_workflow_tools_manage_service.py b/api/tests/e2e_tests/containers/services/tools/test_workflow_tools_manage_service.py index 2244f37893815b..cc1efcdd88f98d 100644 --- a/api/tests/e2e_tests/containers/services/tools/test_workflow_tools_manage_service.py +++ b/api/tests/e2e_tests/containers/services/tools/test_workflow_tools_manage_service.py @@ -5,14 +5,14 @@ import pytest from faker import Faker from pydantic import ValidationError +from sqlalchemy import select from sqlalchemy.orm import Session from core.tools.entities.tool_entities import WorkflowToolParameterConfiguration -from models.tools import WorkflowToolProvider +from models.tools import ToolLabelBinding, WorkflowToolProvider from models.workflow import Workflow as WorkflowModel from models.workflow import WorkflowType from services.app_service import AppService, CreateAppParams -from services.tools.legacy_workflow_tools_manage_service import WorkflowToolManageService from tests.e2e_tests.containers.helpers import accounts as account_fixtures from tests.e2e_tests.containers.helpers import generate_valid_password @@ -28,13 +28,7 @@ def mock_external_service_dependencies(self): patch("services.app_service.EnterpriseService") as mock_enterprise_service, patch("services.app_service.ModelManager.for_tenant") as mock_model_manager, patch("services.account.login_adapters.SystemFeatureService") as mock_account_feature_service, - patch( - "services.tools.legacy_workflow_tools_manage_service.WorkflowToolProviderController" - ) as mock_workflow_tool_provider_controller, - patch("services.tools.legacy_workflow_tools_manage_service.ToolLabelManager") as mock_tool_label_manager, - patch( - "services.tools.legacy_workflow_tools_manage_service.ToolTransformService" - ) as mock_tool_transform_service, + patch("services.tools.workflow_tools_manage_service.ToolTransformService") as mock_tool_transform_service, ): # Setup default mock returns for app service mock_feature_service.is_webapp_auth_enabled.return_value = False @@ -49,12 +43,6 @@ def mock_external_service_dependencies(self): mock_model_instance.get_default_model_instance.return_value = None mock_model_instance.get_default_provider_model_name.return_value = ("openai", "gpt-3.5-turbo") - # Mock WorkflowToolProviderController - mock_workflow_tool_provider_controller.from_db.return_value = None - - # Mock ToolLabelManager - mock_tool_label_manager.update_tool_labels.return_value = None - # Mock ToolTransformService mock_tool_transform_service.workflow_provider_to_controller.return_value = None @@ -63,8 +51,6 @@ def mock_external_service_dependencies(self): "enterprise_service": mock_enterprise_service, "model_manager": mock_model_manager, "account_feature_service": mock_account_feature_service, - "workflow_tool_provider_controller": mock_workflow_tool_provider_controller, - "tool_label_manager": mock_tool_label_manager, "tool_transform_service": mock_tool_transform_service, } @@ -156,7 +142,7 @@ def _create_test_workflow_tool_parameters(self): ] def test_create_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test successful workflow tool creation with valid parameters. @@ -182,10 +168,10 @@ def test_create_workflow_tool_success( tool_description = fake.text(max_nb_chars=200) tool_parameters = self._create_test_workflow_tool_parameters() tool_privacy_policy = fake.text(max_nb_chars=100) - tool_labels = ["automation", "workflow"] + tool_labels = ["productivity", "search"] # Execute the method under test - result = WorkflowToolManageService.create_workflow_tool( + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -226,14 +212,13 @@ def test_create_workflow_tool_success( assert created_tool_provider.app_id == app.id # Verify external service calls - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.assert_called_once() - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called_once() + assert set(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == set(tool_labels) mock_external_service_dependencies[ "tool_transform_service" ].workflow_provider_to_controller.assert_called_once() def test_create_workflow_tool_duplicate_name_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when name already exists. @@ -254,7 +239,7 @@ def test_create_workflow_tool_duplicate_name_error( first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -270,7 +255,7 @@ def test_create_workflow_tool_duplicate_name_error( with pytest.raises( ValueError, match=re.escape(f"Tool with name {first_tool_name} or app_id {app.id} already exists") ) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -297,7 +282,7 @@ def test_create_workflow_tool_duplicate_name_error( assert tool_count == 1 def test_create_workflow_tool_invalid_app_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app does not exist. @@ -319,8 +304,8 @@ def test_create_workflow_tool_invalid_app_error( # Attempt to create workflow tool with non-existent app tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError, match=re.escape(f"App {non_existent_app_id} not found")) as exc_info: - WorkflowToolManageService.create_workflow_tool( + with pytest.raises(ValueError, match="workflow not found or not published") as exc_info: + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=non_existent_app_id, # Non-existent app ID @@ -332,7 +317,7 @@ def test_create_workflow_tool_invalid_app_error( ) # Verify error message - assert f"App {non_existent_app_id} not found" in str(exc_info.value) + assert "workflow not found or not published" in str(exc_info.value) # Verify no workflow tool was created @@ -347,7 +332,7 @@ def test_create_workflow_tool_invalid_app_error( assert tool_count == 0 def test_create_workflow_tool_invalid_parameters_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when parameters are invalid. @@ -366,7 +351,7 @@ def test_create_workflow_tool_invalid_parameters_error( # Attempt to create workflow tool with invalid parameters with pytest.raises(ValidationError) as exc_info: # Setup invalid workflow tool parameters (missing required fields) - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -402,7 +387,7 @@ def test_create_workflow_tool_invalid_parameters_error( assert tool_count == 0 def test_create_workflow_tool_duplicate_app_id_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app_id already exists. @@ -422,7 +407,7 @@ def test_create_workflow_tool_duplicate_app_id_error( # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -439,7 +424,7 @@ def test_create_workflow_tool_duplicate_app_id_error( with pytest.raises( ValueError, match=re.escape(f"Tool with name {second_tool_name} or app_id {app.id} already exists") ) as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, # Same app_id @@ -466,7 +451,7 @@ def test_create_workflow_tool_duplicate_app_id_error( assert tool_count == 1 def test_create_workflow_tool_workflow_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation fails when app has no workflow. @@ -490,8 +475,8 @@ def test_create_workflow_tool_workflow_not_found_error( # Attempt to create workflow tool for app without workflow tool_parameters = self._create_test_workflow_tool_parameters() - with pytest.raises(ValueError, match=re.escape(f"Workflow not found for app {app.id}")) as exc_info: - WorkflowToolManageService.create_workflow_tool( + with pytest.raises(ValueError, match="workflow not found or not published") as exc_info: + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -503,7 +488,7 @@ def test_create_workflow_tool_workflow_not_found_error( ) # Verify error message - assert f"Workflow not found for app {app.id}" in str(exc_info.value) + assert "workflow not found or not published" in str(exc_info.value) # Verify no workflow tool was created tool_count = ( @@ -517,7 +502,7 @@ def test_create_workflow_tool_workflow_not_found_error( assert tool_count == 0 def test_create_workflow_tool_with_human_input_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """A workflow containing Human Input can be published as a tool.""" fake = Faker() @@ -543,7 +528,7 @@ def test_create_workflow_tool_with_human_input_node_success( tool_parameters = self._create_test_workflow_tool_parameters() tool_name = fake.word() - result = WorkflowToolManageService.create_workflow_tool( + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -567,7 +552,7 @@ def test_create_workflow_tool_with_human_input_node_success( assert created_tool.name == tool_name def test_update_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test successful workflow tool update with valid parameters. @@ -589,7 +574,7 @@ def test_update_workflow_tool_success( # Create initial workflow tool initial_tool_name = fake.word() initial_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -618,10 +603,10 @@ def test_update_workflow_tool_success( updated_tool_description = fake.text(max_nb_chars=200) updated_tool_parameters = self._create_test_workflow_tool_parameters() updated_tool_privacy_policy = fake.text(max_nb_chars=100) - updated_tool_labels = ["automation", "updated"] + updated_tool_labels = ["search"] # Execute the update method - result = WorkflowToolManageService.update_workflow_tool( + result = workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -650,12 +635,11 @@ def test_update_workflow_tool_success( assert created_tool.updated_at is not None # Verify external service calls - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.assert_called() - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called() + assert list(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == updated_tool_labels mock_external_service_dependencies["tool_transform_service"].workflow_provider_to_controller.assert_called() def test_update_workflow_tool_with_human_input_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """A workflow tool can be updated after adding a Human Input node.""" fake = Faker() @@ -668,7 +652,7 @@ def test_update_workflow_tool_with_human_input_node_success( # Create initial workflow tool initial_tool_name = fake.word() initial_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -701,7 +685,7 @@ def test_update_workflow_tool_with_human_input_node_success( db_session_with_containers.commit() updated_name = fake.word() - result = WorkflowToolManageService.update_workflow_tool( + result = workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -717,7 +701,7 @@ def test_update_workflow_tool_with_human_input_node_success( assert created_tool.name == updated_name def test_update_workflow_tool_not_found_error( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool update fails when tool does not exist. @@ -740,7 +724,7 @@ def test_update_workflow_tool_not_found_error( # Attempt to update non-existent workflow tool tool_parameters = self._create_test_workflow_tool_parameters() with pytest.raises(ValueError, match=re.escape(f"Tool {non_existent_tool_id} not found")) as exc_info: - WorkflowToolManageService.update_workflow_tool( + workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=non_existent_tool_id, # Non-existent tool ID @@ -767,7 +751,7 @@ def test_update_workflow_tool_not_found_error( assert tool_count == 0 def test_update_workflow_tool_same_name_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool update succeeds when keeping the same name. @@ -787,7 +771,7 @@ def test_update_workflow_tool_same_name_success( # Create first workflow tool first_tool_name = fake.word() first_tool_parameters = self._create_test_workflow_tool_parameters() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -810,7 +794,7 @@ def test_update_workflow_tool_same_name_success( ) # Attempt to update tool with same name (should not fail) - result = WorkflowToolManageService.update_workflow_tool( + result = workflow_tools.update_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_tool_id=created_tool.id, @@ -830,7 +814,7 @@ def test_update_workflow_tool_same_name_success( assert created_tool.updated_at is not None def test_create_workflow_tool_with_file_parameter_default( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation with FILE parameter having a file object as default. @@ -888,8 +872,8 @@ def test_create_workflow_tool_with_file_parameter_default( ] # Execute the method under test - # Note: from_db is mocked, so this test primarily validates the parameter configuration - result = WorkflowToolManageService.create_workflow_tool( + # Note: provider construction is mocked, so this test primarily validates the parameter configuration + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -904,7 +888,7 @@ def test_create_workflow_tool_with_file_parameter_default( assert result == {"result": "success"} def test_create_workflow_tool_with_files_parameter_default( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test workflow tool creation with FILES (Array[File]) parameter having file objects as default. @@ -966,8 +950,8 @@ def test_create_workflow_tool_with_files_parameter_default( ] # Execute the method under test - # Note: from_db is mocked, so this test primarily validates the parameter configuration - result = WorkflowToolManageService.create_workflow_tool( + # Note: provider construction is mocked, so this test primarily validates the parameter configuration + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -982,13 +966,13 @@ def test_create_workflow_tool_with_files_parameter_default( assert result == {"result": "success"} def test_create_workflow_tool_db_commit_before_validation( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """ Test that database commit happens before validation, causing DB pollution on validation failure. This test verifies the second bug: - - WorkflowToolProvider is committed to database BEFORE from_db validation + - WorkflowToolProvider is committed to database BEFORE provider construction validation - If validation fails, the record remains in the database - Subsequent attempts fail with "Tool already exists" error @@ -1004,14 +988,14 @@ def test_create_workflow_tool_db_commit_before_validation( tool_name = fake.word() - # Mock from_db to raise validation error - mock_external_service_dependencies["workflow_tool_provider_controller"].from_db.side_effect = ValueError( - "Validation failed: default parameter type mismatch" - ) + # Mock provider construction to raise validation error + mock_external_service_dependencies[ + "tool_transform_service" + ].workflow_provider_to_controller.side_effect = ValueError("Validation failed: default parameter type mismatch") # Attempt to create workflow tool (will fail at validation stage) with pytest.raises(ValueError, match="Validation failed") as exc_info: - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1043,7 +1027,7 @@ def test_create_workflow_tool_db_commit_before_validation( # assert tool_count == 0 # Expected after fix def test_delete_workflow_tool_success( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test successful deletion of a workflow tool.""" fake = Faker() @@ -1052,7 +1036,7 @@ def test_delete_workflow_tool_success( ) tool_name = fake.unique.word() - WorkflowToolManageService.create_workflow_tool( + workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1070,7 +1054,7 @@ def test_delete_workflow_tool_success( ) assert tool is not None - result = WorkflowToolManageService.delete_workflow_tool(account.id, account.current_tenant.id, tool.id) + result = workflow_tools.delete_workflow_tool(account.id, account.current_tenant.id, tool.id) assert result == {"result": "success"} deleted = ( @@ -1079,7 +1063,7 @@ def test_delete_workflow_tool_success( assert deleted is None def test_list_tenant_workflow_tools_empty( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test listing workflow tools when none exist returns empty list.""" @@ -1087,12 +1071,12 @@ def test_list_tenant_workflow_tools_empty( db_session_with_containers, mock_external_service_dependencies ) - result = WorkflowToolManageService.list_tenant_workflow_tools(account.id, account.current_tenant.id) + result = workflow_tools.list_tenant_workflow_tools(account.id, account.current_tenant.id) assert result == [] def test_get_workflow_tool_by_tool_id_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that get_workflow_tool_by_tool_id raises ValueError when tool not found.""" fake = Faker() @@ -1101,10 +1085,10 @@ def test_get_workflow_tool_by_tool_id_not_found( ) with pytest.raises(ValueError, match="Tool not found"): - WorkflowToolManageService.get_workflow_tool_by_tool_id(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.get_workflow_tool_by_tool_id(account.id, account.current_tenant.id, fake.uuid4()) def test_get_workflow_tool_by_app_id_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that get_workflow_tool_by_app_id raises ValueError when tool not found.""" fake = Faker() @@ -1113,10 +1097,10 @@ def test_get_workflow_tool_by_app_id_not_found( ) with pytest.raises(ValueError, match="Tool not found"): - WorkflowToolManageService.get_workflow_tool_by_app_id(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.get_workflow_tool_by_app_id(account.id, account.current_tenant.id, fake.uuid4()) def test_list_single_workflow_tools_not_found( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): """Test that list_single_workflow_tools raises ValueError when tool not found.""" fake = Faker() @@ -1125,18 +1109,18 @@ def test_list_single_workflow_tools_not_found( ) with pytest.raises(ValueError, match="not found"): - WorkflowToolManageService.list_single_workflow_tools(account.id, account.current_tenant.id, fake.uuid4()) + workflow_tools.list_single_workflow_tools(account.id, account.current_tenant.id, fake.uuid4()) def test_create_workflow_tool_with_labels( - self, db_session_with_containers: Session, mock_external_service_dependencies + self, db_session_with_containers: Session, mock_external_service_dependencies, *, workflow_tools ): - """Test that labels are forwarded to ToolLabelManager when provided.""" + """Labels are persisted atomically with the Workflow Tool.""" fake = Faker() app, account, workflow = self._create_test_app_and_account( db_session_with_containers, mock_external_service_dependencies ) - result = WorkflowToolManageService.create_workflow_tool( + result = workflow_tools.create_workflow_tool( user_id=account.id, tenant_id=account.current_tenant.id, workflow_app_id=app.id, @@ -1145,8 +1129,11 @@ def test_create_workflow_tool_with_labels( icon={"type": "emoji", "emoji": "🔧"}, description=fake.text(max_nb_chars=200), parameters=self._create_test_workflow_tool_parameters(), - labels=["label-1", "label-2"], + labels=["productivity", "search"], ) assert result == {"result": "success"} - mock_external_service_dependencies["tool_label_manager"].update_tool_labels.assert_called_once() + assert set(db_session_with_containers.scalars(select(ToolLabelBinding.label_name))) == { + "productivity", + "search", + } diff --git a/api/tests/e2e_tests/containers/services/workflow/test_workflow_converter.py b/api/tests/e2e_tests/containers/services/workflow/test_workflow_converter.py index 63e2b94d8308c8..cf0dec1e8015f8 100644 --- a/api/tests/e2e_tests/containers/services/workflow/test_workflow_converter.py +++ b/api/tests/e2e_tests/containers/services/workflow/test_workflow_converter.py @@ -1,756 +1,148 @@ -from __future__ import annotations +"""Conversion persists an App aggregate and its draft atomically on PostgreSQL.""" import json -from unittest.mock import MagicMock, patch +from uuid import uuid4 import pytest -from faker import Faker -from sqlalchemy.orm import Session +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker -from core.app.app_config.entities import ( - AdvancedChatMessageEntity, - AdvancedChatPromptTemplateEntity, - AdvancedCompletionPromptTemplateEntity, - DatasetEntity, - DatasetRetrieveConfigEntity, - ExternalDataVariableEntity, - ModelConfigEntity, - PromptTemplateEntity, -) -from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from graphon.model_runtime.entities.llm_entities import LLMMode -from graphon.model_runtime.entities.message_entities import PromptMessageRole -from graphon.variables.input_entities import VariableEntity, VariableEntityType -from models import Account, Tenant -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import App, AppMode, AppModelConfig +from machinery.context import RequestContext +from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole +from models.model import App, AppMode, AppModelConfig, InstalledApp, Site from models.workflow import Workflow +from repositories.app.console_repository import ConsoleAppRepository +from services.entities.app_entities import AppEvent +from services.workflow.conversion_service import WorkflowConversionService from services.workflow.workflow_converter import WorkflowConverter +type ConversionSource = tuple[sessionmaker[Session], RequestContext, str] -class TestWorkflowConverter: - """Integration tests for WorkflowConverter using testcontainers.""" - @pytest.fixture - def mock_external_service_dependencies(self): - """Mock setup for external service dependencies.""" - with ( - patch("services.workflow.workflow_converter.encrypter") as mock_encrypter, - patch("services.workflow.workflow_converter.SimplePromptTransform") as mock_prompt_transform, - patch("services.workflow.workflow_converter.AgentChatAppConfigManager") as mock_agent_chat_config_manager, - patch("services.workflow.workflow_converter.ChatAppConfigManager") as mock_chat_config_manager, - patch("services.workflow.workflow_converter.CompletionAppConfigManager") as mock_completion_config_manager, - ): - # Setup default mock returns - mock_encrypter.decrypt_token.return_value = "decrypted_api_key" - mock_prompt_transform.return_value.get_prompt_template.return_value = { - "prompt_template": PromptTemplateParser(template="You are a helpful assistant {{text_input}}"), - "prompt_rules": {"human_prefix": "Human", "assistant_prefix": "Assistant"}, - } - mock_agent_chat_config_manager.get_app_config.return_value = self._create_mock_app_config() - mock_chat_config_manager.get_app_config.return_value = self._create_mock_app_config() - mock_completion_config_manager.get_app_config.return_value = self._create_mock_app_config() - - yield { - "encrypter": mock_encrypter, - "prompt_transform": mock_prompt_transform, - "agent_chat_config_manager": mock_agent_chat_config_manager, - "chat_config_manager": mock_chat_config_manager, - "completion_config_manager": mock_completion_config_manager, - } - - def _create_mock_app_config(self): - """Helper method to create a mock app config.""" - mock_config = type("obj", (object,), {})() - mock_config.variables = [ - VariableEntity( - variable="text_input", - label="Text Input", - type=VariableEntityType.TEXT_INPUT, - ) - ] - mock_config.model = ModelConfigEntity( - provider="openai", - model="gpt-4", - mode=LLMMode.CHAT, - parameters={}, - stop=[], - ) - mock_config.prompt_template = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are a helpful assistant {{text_input}}", - ) - mock_config.dataset = None - mock_config.external_data_variables = [] - mock_config.additional_features = type("obj", (object,), {"file_upload": None})() - mock_config.app_model_config_dict = {} - return mock_config - - def _create_test_account_and_tenant(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Helper method to create a test account and tenant for testing. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - mock_external_service_dependencies: Mock dependencies - - Returns: - tuple: (account, tenant) - Created account and tenant instances - """ - fake = Faker() - - # Create account - account = Account( - email=fake.email(), - name=fake.name(), - interface_language="en-US", - status="active", - ) - - db_session_with_containers.add(account) - db_session_with_containers.commit() - - # Create tenant for the account - tenant = Tenant( - name=fake.company(), - status="normal", - ) - db_session_with_containers.add(tenant) - db_session_with_containers.commit() - - # Create tenant-account join - from models.account import TenantAccountJoin, TenantAccountRole - - join = TenantAccountJoin( - tenant_id=tenant.id, - account_id=account.id, - role=TenantAccountRole.OWNER, - current=True, - ) - db_session_with_containers.add(join) - db_session_with_containers.commit() - - # Set current tenant for account - account.set_current_tenant_with_session(tenant, session=db_session_with_containers) - - return account, tenant - - def _create_test_app( - self, db_session_with_containers: Session, mock_external_service_dependencies, tenant, account - ): - """ - Helper method to create a test app for testing. - - Args: - db_session_with_containers: Database session from testcontainers infrastructure - mock_external_service_dependencies: Mock dependencies - tenant: Tenant instance - account: Account instance - - Returns: - App: Created app instance - """ - fake = Faker() - - # Create app - app = App( - tenant_id=tenant.id, - name=fake.company(), - mode=AppMode.CHAT, - icon_type="emoji", - icon="🤖", - icon_background="#FF6B6B", - enable_site=True, - enable_api=True, - api_rpm=100, - api_rph=10, - is_demo=False, - is_public=False, - created_by=account.id, - updated_by=account.id, - ) - - db_session_with_containers.add(app) - db_session_with_containers.commit() - - # Create app model config - app_model_config = AppModelConfig( - app_id=app.id, - provider="openai", - model="gpt-4", - configs={}, - created_by=account.id, - updated_by=account.id, - ) - db_session_with_containers.add(app_model_config) - db_session_with_containers.commit() - - # Link app model config to app - app.app_model_config_id = app_model_config.id - db_session_with_containers.commit() - - return app - - def test_convert_to_workflow_success(self, db_session_with_containers: Session, mock_external_service_dependencies): - """ - Test successful conversion of app to workflow. - - This test verifies: - - Proper app to workflow conversion - - Correct database state after conversion - - Proper relationship establishment - - Workflow creation with correct configuration - """ - # Arrange: Create test data - - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - new_app = workflow_converter.convert_to_workflow( - app_model=app, - account=account, - name="Test Workflow App", - icon_type="emoji", - icon="🚀", - icon_background="#4CAF50", - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes - assert new_app is not None - assert new_app.name == "Test Workflow App" - assert new_app.mode == AppMode.ADVANCED_CHAT - assert new_app.icon_type == "emoji" - assert new_app.icon == "🚀" - assert new_app.icon_background == "#4CAF50" - assert new_app.tenant_id == app.tenant_id - assert new_app.created_by == account.id - - # Verify database state - - db_session_with_containers.refresh(new_app) - assert new_app.id is not None - - # Verify workflow was created - workflow = db_session_with_containers.query(Workflow).where(Workflow.app_id == new_app.id).first() - assert workflow is not None - assert workflow.tenant_id == app.tenant_id - assert workflow.type == "chat" - - def test_convert_to_workflow_without_app_model_config_error( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test error handling when app model config is missing. - - This test verifies: - - Proper error handling for missing app model config - - Correct exception type and message - - Database state remains unchanged - """ - # Arrange: Create test data without app model config - fake = Faker() - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - app = App( - tenant_id=tenant.id, - name=fake.company(), - mode=AppMode.CHAT, - icon_type="emoji", - icon="🤖", - icon_background="#FF6B6B", - enable_site=True, - enable_api=True, - api_rpm=100, - api_rph=10, - is_demo=False, - is_public=False, - created_by=account.id, - updated_by=account.id, - ) - - db_session_with_containers.add(app) - db_session_with_containers.commit() - - # Act & Assert: Verify proper error handling - workflow_converter = WorkflowConverter() - - # Check initial state - initial_workflow_count = db_session_with_containers.query(Workflow).count() - - with pytest.raises(ValueError, match="App model config is required"): - workflow_converter.convert_to_workflow( - app_model=app, - account=account, - name="Test Workflow App", - icon_type="emoji", - icon="🚀", - icon_background="#4CAF50", - session=db_session_with_containers, - ) - - # Verify database state remains unchanged - # The workflow creation happens in convert_app_model_config_to_workflow - # which is called before the app_model_config check, so we need to clean up - db_session_with_containers.rollback() - final_workflow_count = db_session_with_containers.query(Workflow).count() - assert final_workflow_count == initial_workflow_count - - def test_convert_app_model_config_to_workflow_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion of app model config to workflow. - - This test verifies: - - Proper app model config to workflow conversion - - Correct workflow graph structure - - Proper node creation and configuration - - Database state management - """ - # Arrange: Create test data - - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - workflow = workflow_converter.convert_app_model_config_to_workflow( - app_model=app, - app_model_config=app.app_model_config_with_session(session=db_session_with_containers), - account_id=account.id, - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes +@pytest.fixture +def conversion_source(db_session_with_containers: Session) -> ConversionSource: + class ConversionSession(Session): + pass + + sessions: sessionmaker[Session] = sessionmaker[Session]( + db_session_with_containers.get_bind(), class_=ConversionSession, expire_on_commit=False + ) + account_id, tenant_id, app_id, config_id = (str(uuid4()) for _ in range(4)) + with sessions.begin() as session: + session.add_all( + [ + (account := Account(name="Converter", email=f"{account_id}@example.com", interface_language="en-US")), + (tenant := Tenant(name="Conversion workspace")), + ] + ) + account.id = account_id + tenant.id = tenant_id + session.flush() + session.add_all( + [ + TenantAccountJoin( + tenant_id=tenant_id, account_id=account_id, role=TenantAccountRole.OWNER, current=True + ), + App( + id=app_id, + tenant_id=tenant_id, + name="Source", + mode=AppMode.CHAT, + app_model_config_id=config_id, + enable_api=True, + enable_site=True, + created_by=account_id, + updated_by=account_id, + ), + ( + config := AppModelConfig( + app_id=app_id, + model=json.dumps( + {"provider": "openai", "name": "gpt-4", "mode": "chat", "completion_params": {}} + ), + pre_prompt="Hello {{name}}", + user_input_form=json.dumps([{"text-input": {"variable": "name", "label": "Name"}}]), + ) + ), + ] + ) + config.id = config_id + return sessions, RequestContext("conversion", None, account_id, tenant_id), app_id + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) +def test_conversion_creates_complete_target_before_notification( + conversion_source: ConversionSource, mode: AppMode +) -> None: + sessions, context, app_id = conversion_source + with sessions.begin() as session: + app = session.get(App, app_id) + assert app is not None + app.mode = mode + observed: list[str] = [] + + def created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: + assert (account_id, backing_agent_id) == (context.account_id, None) + with sessions() as session: + assert session.scalar(select(Workflow).where(Workflow.app_id == event.id)) is not None + assert session.scalar(select(Site).where(Site.app_id == event.id)) is not None + assert session.scalar(select(InstalledApp).where(InstalledApp.app_id == event.id)) is not None + observed.append(event.id) + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=created, + ) + new_id = service.convert(context, app_id, {"name": "Converted", "icon": "🚀"}) + assert observed == [new_id] + with sessions() as session: + app = session.get(App, new_id) + assert app is not None + assert app.name == "Converted" + assert app.icon == "🚀" + assert app.mode == (AppMode.ADVANCED_CHAT if mode == AppMode.CHAT else AppMode.WORKFLOW) + assert app.created_by == context.account_id + assert session.scalar(select(Workflow.id).where(Workflow.app_id == app_id)) is None + workflow = session.scalar(select(Workflow).where(Workflow.app_id == new_id)) assert workflow is not None - assert workflow.tenant_id == app.tenant_id - assert workflow.app_id == app.id - assert workflow.type == "chat" assert workflow.version == Workflow.VERSION_DRAFT - assert workflow.created_by == account.id - - # Verify workflow graph structure - graph = json.loads(workflow.graph) - assert "nodes" in graph - assert "edges" in graph - assert len(graph["nodes"]) > 0 - assert len(graph["edges"]) > 0 - - # Verify start node exists - start_node = next((node for node in graph["nodes"] if node["data"]["type"] == "start"), None) - assert start_node is not None - assert start_node["id"] == "start" - - # Verify LLM node exists - llm_node = next((node for node in graph["nodes"] if node["data"]["type"] == "llm"), None) - assert llm_node is not None - assert llm_node["id"] == "llm" - - # Verify answer node exists for chat mode - answer_node = next((node for node in graph["nodes"] if node["data"]["type"] == "answer"), None) - assert answer_node is not None - assert answer_node["id"] == "answer" - - # Verify database state - - db_session_with_containers.refresh(workflow) - assert workflow.id is not None - - # Verify features were set - features = json.loads(workflow._features) if workflow._features else {} - assert isinstance(features, dict) - - def test_convert_to_start_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to start node. - - This test verifies: - - Proper start node creation with variables - - Correct node structure and data - - Variable encoding and formatting - """ - # Arrange: Create test variables - variables = [ - VariableEntity( - variable="text_input", - label="Text Input", - type=VariableEntityType.TEXT_INPUT, - ), - VariableEntity( - variable="number_input", - label="Number Input", - type=VariableEntityType.NUMBER, - ), - ] - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - start_node = workflow_converter._convert_to_start_node(variables=variables) - - # Assert: Verify the expected outcomes - assert start_node is not None - assert start_node["id"] == "start" - assert start_node["data"]["title"] == "START" - assert start_node["data"]["type"] == "start" - assert len(start_node["data"]["variables"]) == 2 - - # Verify variable encoding - first_variable = start_node["data"]["variables"][0] - assert first_variable["variable"] == "text_input" - assert first_variable["label"] == "Text Input" - assert first_variable["type"] == "text-input" - - second_variable = start_node["data"]["variables"][1] - assert second_variable["variable"] == "number_input" - assert second_variable["label"] == "Number Input" - assert second_variable["type"] == "number" - - def test_convert_to_http_request_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to HTTP request node. - - This test verifies: - - Proper HTTP request node creation - - Correct API configuration and authorization - - Code node creation for response parsing - - External data variable mapping - """ - # Arrange: Create test data - - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - app = self._create_test_app(db_session_with_containers, mock_external_service_dependencies, tenant, account) - - # Create API based extension - api_based_extension = APIBasedExtension( - tenant_id=tenant.id, - name="Test API Extension", - api_key="encrypted_api_key", - api_endpoint="https://api.example.com/test", - ) - - db_session_with_containers.add(api_based_extension) - db_session_with_containers.commit() - - # Mock encrypter - mock_external_service_dependencies["encrypter"].decrypt_token.return_value = "decrypted_api_key" - - variables = [ - VariableEntity( - variable="user_input", - label="User Input", - type=VariableEntityType.TEXT_INPUT, - ) - ] - - external_data_variables = [ - ExternalDataVariableEntity( - variable="external_data", type="api", config={"api_based_extension_id": api_based_extension.id} - ) - ] - - # Act: Execute the conversion - workflow_converter = WorkflowConverter() - nodes, external_data_variable_node_mapping = workflow_converter._convert_to_http_request_node( - app_model=app, - variables=variables, - external_data_variables=external_data_variables, - session=db_session_with_containers, - ) - - # Assert: Verify the expected outcomes - assert len(nodes) == 2 # HTTP request node + code node - assert len(external_data_variable_node_mapping) == 1 - - # Verify HTTP request node - http_request_node = nodes[0] - assert http_request_node["data"]["type"] == "http-request" - assert http_request_node["data"]["method"] == "post" - assert http_request_node["data"]["url"] == api_based_extension.api_endpoint - assert http_request_node["data"]["authorization"]["type"] == "api-key" - assert http_request_node["data"]["authorization"]["config"]["type"] == "bearer" - assert http_request_node["data"]["authorization"]["config"]["api_key"] == "decrypted_api_key" - - # Verify code node - code_node = nodes[1] - assert code_node["data"]["type"] == "code" - assert code_node["data"]["code_language"] == "python3" - assert "response_json" in code_node["data"]["variables"][0]["variable"] - - # Verify mapping - assert external_data_variable_node_mapping["external_data"] == code_node["id"] - - def test_convert_to_knowledge_retrieval_node_success( - self, db_session_with_containers: Session, mock_external_service_dependencies - ): - """ - Test successful conversion to knowledge retrieval node. - - This test verifies: - - Proper knowledge retrieval node creation - - Correct dataset configuration - - Model configuration integration - - Query variable selector setup - """ - # Arrange: Create test data - - account, tenant = self._create_test_account_and_tenant( - db_session_with_containers, mock_external_service_dependencies - ) - - # Create dataset config - dataset_config = DatasetEntity( - dataset_ids=["dataset_1", "dataset_2"], - retrieve_config=DatasetRetrieveConfigEntity( - retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE, - top_k=10, - score_threshold=0.8, - reranking_model={"reranking_provider_name": "cohere", "reranking_model_name": "rerank-v2"}, - reranking_enabled=True, - ), - ) - - model_config = ModelConfigEntity( - provider="openai", - model="gpt-4", - mode=LLMMode.CHAT, - parameters={"temperature": 0.7}, - stop=[], - ) - - # Act: Execute the conversion for advanced chat mode - workflow_converter = WorkflowConverter() - node = workflow_converter._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.ADVANCED_CHAT, - dataset_config=dataset_config, - model_config=model_config, - ) - - # Assert: Verify the expected outcomes - assert node is not None - assert node["data"]["type"] == "knowledge-retrieval" - assert node["data"]["title"] == "KNOWLEDGE RETRIEVAL" - assert node["data"]["dataset_ids"] == ["dataset_1", "dataset_2"] - assert node["data"]["retrieval_mode"] == "multiple" - assert node["data"]["query_variable_selector"] == ["sys", "query"] - - # Verify multiple retrieval config - multiple_config = node["data"]["multiple_retrieval_config"] - assert multiple_config["top_k"] == 10 - assert multiple_config["score_threshold"] == 0.8 - assert multiple_config["reranking_model"]["reranking_provider_name"] == "cohere" - assert multiple_config["reranking_model"]["reranking_model_name"] == "rerank-v2" - - # Verify single retrieval config is None for multiple strategy - assert node["data"]["single_retrieval_config"] is None - - -@pytest.fixture -def default_variables(): - return [ - VariableEntity(variable="text_input", label="text-input", type=VariableEntityType.TEXT_INPUT), - VariableEntity(variable="paragraph", label="paragraph", type=VariableEntityType.PARAGRAPH), - VariableEntity(variable="select", label="select", type=VariableEntityType.SELECT), - ] - - -class TestConvertToHttpRequestNodeVariants: - """Tests for chatbot vs workflow differences in HTTP request node conversion.""" - - @staticmethod - def _setup(app_mode, default_variables, db_session_with_containers: Session): - app_model = App( - tenant_id="tenant_id", - mode=app_mode, - name="test", - icon_type="emoji", - icon="🤖", - icon_background="#FFFFFF", - ) - - ext = APIBasedExtension(tenant_id="tenant_id", name="api-1", api_key="enc", api_endpoint="https://dify.ai") - ext.id = "ext_id" - - converter = WorkflowConverter() - converter._get_api_based_extension = MagicMock(return_value=ext) - - from core.helper import encrypter - - encrypter.decrypt_token = MagicMock(return_value="api_key") - - ext_vars = [ - ExternalDataVariableEntity( - variable="external_variable", type="api", config={"api_based_extension_id": "ext_id"} - ) + assert [node["id"] for node in json.loads(workflow.graph)["nodes"]] == [ + "start", + "llm", + "answer" if mode == AppMode.CHAT else "end", ] - nodes, _ = converter._convert_to_http_request_node( - app_model=app_model, - variables=default_variables, - external_data_variables=ext_vars, - session=db_session_with_containers, - ) - return nodes - - def test_chatbot_query_uses_sys_query(self, default_variables, db_session_with_containers: Session): - nodes = self._setup(AppMode.CHAT, default_variables, db_session_with_containers) - - body = json.loads(nodes[0]["data"]["body"]["data"]) - assert body["params"]["query"] == "{{#sys.query#}}" - assert body["point"] == APIBasedExtensionPoint.APP_EXTERNAL_DATA_TOOL_QUERY - assert nodes[1]["data"]["type"] == "code" - def test_workflow_query_is_empty(self, default_variables, db_session_with_containers: Session): - nodes = self._setup(AppMode.WORKFLOW, default_variables, db_session_with_containers) - body = json.loads(nodes[0]["data"]["body"]["data"]) - assert body["params"]["query"] == "" - - -class TestConvertToKnowledgeRetrievalNodeVariants: - """Tests for chatbot vs workflow differences in knowledge retrieval node.""" - - @staticmethod - def _dataset_config(query_variable=None): - return DatasetEntity( - dataset_ids=["ds1", "ds2"], - retrieve_config=DatasetRetrieveConfigEntity( - query_variable=query_variable, - retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE, - top_k=5, - score_threshold=0.8, - reranking_model={"reranking_provider_name": "cohere", "reranking_model_name": "rerank-english-v2.0"}, - reranking_enabled=True, - ), - ) - - @staticmethod - def _model_config(): - return ModelConfigEntity(provider="openai", model="gpt-4", mode="chat", parameters={}, stop=[]) - - def test_chatbot_uses_sys_query(self): - node = WorkflowConverter()._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.ADVANCED_CHAT, - dataset_config=self._dataset_config(), - model_config=self._model_config(), - ) - assert node["data"]["query_variable_selector"] == ["sys", "query"] - - def test_workflow_uses_start_variable(self): - node = WorkflowConverter()._convert_to_knowledge_retrieval_node( - new_app_mode=AppMode.WORKFLOW, - dataset_config=self._dataset_config(query_variable="query"), - model_config=self._model_config(), - ) - assert node["data"]["query_variable_selector"] == ["start", "query"] - - -class TestConvertToLlmNode: - """Tests for LLM node conversion across model modes and prompt types.""" - - @staticmethod - def _model_config(model, mode): - return ModelConfigEntity( - provider="openai", - model=model, - mode=mode.value, - parameters={}, - stop=[], - ) - - @staticmethod - def _graph(default_variables): - start = WorkflowConverter()._convert_to_start_node(default_variables) - return {"nodes": [start], "edges": []} - - def test_simple_chat_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are helpful {{text_input}}, {{paragraph}}, {{select}}.", - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-4", LLMMode.CHAT), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert node["data"]["type"] == "llm" - assert node["data"]["model"]["mode"] == LLMMode.CHAT.value - assert node["data"]["context"]["enabled"] is False - expected = "You are helpful {{#start.text_input#}}, {{#start.paragraph#}}, {{#start.select#}}.\n" - assert node["data"]["prompt_template"][0]["text"] == expected - - def test_simple_completion_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.SIMPLE, - simple_prompt_template="You are helpful {{text_input}}, {{paragraph}}, {{select}}.", - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-3.5-turbo-instruct", LLMMode.COMPLETION), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert node["data"]["model"]["mode"] == LLMMode.COMPLETION.value - expected = "You are helpful {{#start.text_input#}}, {{#start.paragraph#}}, {{#start.select#}}.\n" - assert node["data"]["prompt_template"]["text"] == expected - - def test_advanced_chat_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.ADVANCED, - advanced_chat_prompt_template=AdvancedChatPromptTemplateEntity( - messages=[ - AdvancedChatMessageEntity( - text="You are helpful named {{name}}.\n\nContext:\n{{#context#}}", - role=PromptMessageRole.SYSTEM, - ), - AdvancedChatMessageEntity(text="Hi.", role=PromptMessageRole.USER), - AdvancedChatMessageEntity(text="Hello!", role=PromptMessageRole.ASSISTANT), - ] - ), - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-4", LLMMode.CHAT), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert isinstance(node["data"]["prompt_template"], list) - assert len(node["data"]["prompt_template"]) == 3 - - def test_advanced_completion_model(self, default_variables): - prompt = PromptTemplateEntity( - prompt_type=PromptTemplateEntity.PromptType.ADVANCED, - advanced_completion_prompt_template=AdvancedCompletionPromptTemplateEntity( - prompt="You are helpful named {{name}}.\n\nContext:\n{{#context#}}\n\nHuman: hi\nAssistant: ", - role_prefix=AdvancedCompletionPromptTemplateEntity.RolePrefixEntity( - user="Human", assistant="Assistant" - ), - ), - ) - node = WorkflowConverter()._convert_to_llm_node( - original_app_mode=AppMode.CHAT, - new_app_mode=AppMode.ADVANCED_CHAT, - model_config=self._model_config("gpt-3.5-turbo-instruct", LLMMode.COMPLETION), - graph=self._graph(default_variables), - prompt_template=prompt, - ) - assert isinstance(node["data"]["prompt_template"], dict) - assert "text" in node["data"]["prompt_template"] +def test_workflow_write_failure_rolls_back_target_and_required_records( + conversion_source: ConversionSource, +) -> None: + sessions, context, app_id = conversion_source + + def reject_workflow(session: Session, *_: object) -> None: + if any(isinstance(row, Workflow) for row in session.new): + raise RuntimeError("workflow write failed") + + event.listen(sessions.class_, "before_flush", reject_workflow) + notifications: list[dict[str, object]] = [] + + def notify_created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: + notifications.append({"event": event, "account_id": account_id, "backing_agent_id": backing_agent_id}) + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=notify_created, + ) + with pytest.raises(RuntimeError, match="workflow write failed"): + service.convert(context, app_id, {}) + assert notifications == [] + with sessions() as session: + assert list(session.scalars(select(App.id).where(App.tenant_id == context.active_workspace_id))) == [app_id] + assert session.scalar(select(Workflow.id).where(Workflow.tenant_id == context.active_workspace_id)) is None + assert ( + session.scalar(select(InstalledApp.id).where(InstalledApp.tenant_id == context.active_workspace_id)) is None + ) + assert session.scalar(select(Site.id).where(Site.created_by == context.account_id)) is None diff --git a/api/tests/e2e_tests/containers/services/workflow/test_workflow_deletion.py b/api/tests/e2e_tests/containers/services/workflow/test_workflow_deletion.py index 24317df36297c2..cf8830d94b54cf 100644 --- a/api/tests/e2e_tests/containers/services/workflow/test_workflow_deletion.py +++ b/api/tests/e2e_tests/containers/services/workflow/test_workflow_deletion.py @@ -1,19 +1,19 @@ -"""Testcontainers integration tests for WorkflowService.delete_workflow.""" +"""Testcontainers integration tests for WorkflowDefinitionStore.delete_workflow.""" import json from uuid import uuid4 import pytest -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session -from extensions.ext_database import db from models.account import Account, Tenant, TenantAccountJoin from models.model import App from models.tools import WorkflowToolProvider from models.workflow import Workflow from models.workflow import WorkflowType as ModelWorkflowType +from repositories.workflow.definition_repository import WorkflowDefinitionStore +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError from services.workflow_ref_service import WorkflowRef -from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService class TestWorkflowDeletion: @@ -111,13 +111,13 @@ def test_delete_workflow_success(self, db_session_with_containers: Session): db_session_with_containers.commit() workflow_id = workflow.id - service = WorkflowService(sessionmaker(bind=db.engine)) - result = service.delete_workflow( + result = WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow_id), ) assert result == [] + db_session_with_containers.flush() db_session_with_containers.expire_all() assert db_session_with_containers.get(Workflow, workflow_id) is None @@ -129,9 +129,8 @@ def test_delete_draft_workflow_raises_error(self, db_session_with_containers: Se ) db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(DraftWorkflowDeletionError): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) @@ -146,9 +145,8 @@ def test_delete_workflow_in_use_by_app_raises_error(self, db_session_with_contai app.workflow_id = workflow.id db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(WorkflowInUseError, match="currently in use by app"): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) @@ -162,9 +160,8 @@ def test_delete_workflow_published_as_tool_raises_error(self, db_session_with_co self._create_tool_provider(db_session_with_containers, tenant=tenant, app=app, account=account, version="1.0") db_session_with_containers.commit() - service = WorkflowService(sessionmaker(bind=db.engine)) with pytest.raises(WorkflowInUseError, match="published as a tool"): - service.delete_workflow( + WorkflowDefinitionStore.delete_workflow( session=db_session_with_containers, workflow_ref=WorkflowRef(tenant_id=tenant.id, owner_id=app.id, workflow_id=workflow.id), ) diff --git a/api/tests/e2e_tests/containers/tasks/test_mail_human_input_delivery_task.py b/api/tests/e2e_tests/containers/tasks/test_mail_human_input_delivery_task.py index 2a86dd32197f68..a049eb618eed22 100644 --- a/api/tests/e2e_tests/containers/tasks/test_mail_human_input_delivery_task.py +++ b/api/tests/e2e_tests/containers/tasks/test_mail_human_input_delivery_task.py @@ -10,13 +10,13 @@ from core.app.app_config.entities import WorkflowUIBasedAppConfig from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext -from core.repositories.human_input_repository import FormCreateParams, HumanInputFormRepositoryImpl from extensions.ext_storage import storage from graphon.enums import WorkflowExecutionStatus from graphon.runtime import RuntimeState, VariablePool from models.account import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAccountRole from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient +from models.human_input_contracts import FormCreateParams from models.human_input_delivery import ( EmailDeliveryConfig, EmailDeliveryMethod, @@ -27,6 +27,7 @@ from models.human_input_entities import HumanInputNodeData from models.model import AppMode from models.workflow import WorkflowPause, WorkflowRun, WorkflowType +from repositories.human_input.form_repository import HumanInputFormRepositoryImpl from tasks.mail_human_input_delivery_task import dispatch_human_input_email_task diff --git a/api/tests/e2e_tests/containers/tasks/test_rag_pipeline_run_tasks.py b/api/tests/e2e_tests/containers/tasks/test_rag_pipeline_run_tasks.py index ce7ee8669e0c57..a8e8e72a35da00 100644 --- a/api/tests/e2e_tests/containers/tasks/test_rag_pipeline_run_tasks.py +++ b/api/tests/e2e_tests/containers/tasks/test_rag_pipeline_run_tasks.py @@ -38,7 +38,9 @@ class TestRagPipelineRunTasks: @pytest.fixture def mock_pipeline_generator(self): """Mock PipelineGenerator._generate method.""" - with patch("core.app.apps.pipeline.pipeline_generator.PipelineGenerator._generate") as mock_generate: + with patch( + "services.workflow.execution.adapters.pipeline.pipeline_generator.PipelineGenerator._generate" + ) as mock_generate: # Mock the _generate method to return a simple response mock_generate.return_value = {"answer": "Test response", "metadata": {"test": "data"}} yield mock_generate diff --git a/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py b/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py index 9519c114da793b..5bce76d7eef1d1 100644 --- a/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py +++ b/api/tests/e2e_tests/containers/trigger/test_trigger_e2e.py @@ -11,7 +11,7 @@ from flask import Flask, Response from flask.testing import FlaskClient from sqlalchemy import select -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from configs import dify_config from core.plugin.entities.request import TriggerInvokeEventResponse @@ -39,6 +39,7 @@ WorkflowWebhookTrigger, ) from models.workflow import Workflow +from repositories.credentials.query_repository import CredentialQueryRepository from schedule import workflow_schedule_task from schedule.workflow_schedule_task import poll_workflow_schedules from services.system_feature_service import SystemFeatureService @@ -121,7 +122,13 @@ def test_publish_blocks_start_and_trigger_coexistence( ) with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): - workflow_service.publish_workflow(session=db_session_with_containers, app_model=app_model, account=account) + workflow_service.validate_publication( + app_model, + draft_workflow, + credentials=CredentialQueryRepository( + session_factory=sessionmaker(bind=db_session_with_containers.get_bind()) + ), + ) def test_trigger_url_uses_config_base(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py b/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py index 87b21be992eecf..71e739c369686d 100644 --- a/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py +++ b/api/tests/e2e_tests/contracts/controllers/console/app/test_message_suggested_questions.py @@ -18,6 +18,8 @@ from enums import DeploymentEdition from extensions.application_services.app import AppServices, build_app_services from extensions.application_services.rbac import RBACServices +from extensions.application_services.workflow import build_console_workflow_service +from extensions.application_services.workflow_variables import build_workflow_variable_service from extensions.ext_application_services import _build_oauth_server_service from extensions.ext_database import db from extensions.ext_redis import RedisClientWrapper @@ -45,6 +47,8 @@ from repositories.app.tracing_config_repository import SQLAlchemyAppTracingConfigRepository from repositories.message_repository import MessageRepository from repositories.recommended_app_catalog_repository import DatabaseRecommendedAppCatalogRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.agent.roster_package_exporter import RosterAgentPackageExporter from services.app.agent_app_service import AgentAppAccessService from services.app_tracing_config_gateway import OpsTraceManagerGateway @@ -181,6 +185,8 @@ def track_session(session: Session, _transaction: SessionTransaction, _connectio monkeypatch, LOGIN_DISABLED=True, RBAC_ENABLED=False, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY ) redis = RedisClientWrapper() + workflow_tools = WorkflowToolRepository(sqlite_session_factory) + workflow_variables = build_workflow_variable_service(database_client=sqlite_session_factory) app_services = build_app_services( database_client=sqlite_session_factory, repository=ConsoleAppRepository(session_factory=sqlite_session_factory), @@ -194,6 +200,14 @@ def track_session(session: Session, _transaction: SessionTransaction, _connectio sources=DatabaseRecommendedAppCatalogRepository(sqlite_session_factory, redis=redis), exporter=RosterAgentPackageExporter(), ), + tool_providers=ToolProviderRepository(sqlite_session_factory), + workflow_tools=workflow_tools, + workflow_publisher=build_console_workflow_service( + database_client=sqlite_session_factory, + redis=redis, + variables=workflow_variables, + rbac_members=rbac_domain.rbac.members, + ), ) flask_app = Flask(__name__) flask_app.config.update(TESTING=True, RESTX_ERROR_404_HELP=False, SQLALCHEMY_DATABASE_URI=str(sqlite_engine.url)) diff --git a/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py b/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py index f26ba315481f07..5b7027f9f74eb3 100644 --- a/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py +++ b/api/tests/e2e_tests/contracts/controllers/service_api/test_conftest.py @@ -9,6 +9,7 @@ persist_service_api_dataset_owner, persist_service_api_tenant_owner, ) +from tests.unit_tests.conftest import workflow_variables as workflow_variables # noqa: PLC0414 from tests.unit_tests.controllers.conftest import app_query_services as app_query_services # noqa: PLC0414 from tests.unit_tests.controllers.conftest import ( datasource_application_dependencies as datasource_application_dependencies, # noqa: PLC0414 diff --git a/api/tests/e2e_tests/contracts/core/workflow/test_node_mapping_bootstrap.py b/api/tests/e2e_tests/contracts/core/workflow/test_node_mapping_bootstrap.py index c18cc2218c5790..3870e19cd76cbe 100644 --- a/api/tests/e2e_tests/contracts/core/workflow/test_node_mapping_bootstrap.py +++ b/api/tests/e2e_tests/contracts/core/workflow/test_node_mapping_bootstrap.py @@ -101,17 +101,18 @@ def test_moved_core_nodes_resolve_after_importing_production_entrypoints() -> No env["PYTHONSAFEPATH"] = "1" script = textwrap.dedent( """ - from core.app.apps import workflow_app_runner + from services.workflow.execution.adapters import graph as workflow_app_runner + from services.workflow.execution.adapters import workflow_entry from core.workflow.nodes.datasource.datasource_node import DatasourceNode from core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPE from core.workflow.nodes.knowledge_index.knowledge_index_node import KnowledgeIndexNode - from services.workflow.execution.adapters import workflow_entry from services.workflow.execution.adapters.knowledge_retrieval import KnowledgeRetrievalNode from services.workflow.execution.adapters.node_factory import ( DifyNodeFactory, NODE_TYPE_CLASSES_MAPPING, resolve_workflow_node_class, ) + from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from graphon.enums import BuiltinNodeTypes from services import workflow_service from services.rag_pipeline import rag_pipeline @@ -131,6 +132,20 @@ def test_moved_core_nodes_resolve_after_importing_production_entrypoints() -> No resolved = DifyNodeFactory._resolve_node_class(node_type=node_type, node_version="1") assert resolved is node_class, (node_type, resolved) + # Resolve through the production bootstrap before importing the moved + # classes; preloading them here would hide missing registration. + agent = DifyNodeFactory._resolve_node_class(node_type=BuiltinNodeTypes.AGENT, node_version="1") + webhook = DifyNodeFactory._resolve_node_class(node_type=TRIGGER_WEBHOOK_NODE_TYPE, node_version="1") + from services.workflow.execution.adapters.agent_node import AgentNode + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode + from services.workflow.execution.adapters.trigger_webhook import TriggerWebhookNode + + from services.workflow.execution.adapters.knowledge_retrieval import KnowledgeRetrievalNode + assert DifyNodeFactory._resolve_node_class( + node_type=BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL, node_version="1" + ) is KnowledgeRetrievalNode + assert agent is AgentNode + assert webhook is TriggerWebhookNode assert DifyNodeFactory._resolve_node_class( node_type=BuiltinNodeTypes.AGENT, node_version="2", diff --git a/api/tests/e2e_tests/middleware/workflow/nodes/test_code.py b/api/tests/e2e_tests/middleware/workflow/nodes/test_code.py index 901cf3aa7db13d..a7fe7f66a1c8c2 100644 --- a/api/tests/e2e_tests/middleware/workflow/nodes/test_code.py +++ b/api/tests/e2e_tests/middleware/workflow/nodes/test_code.py @@ -7,15 +7,14 @@ from configs import dify_config from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom -from core.workflow.node_factory import DifyNodeFactory from core.workflow.system_variables import build_system_variables from graphon.enums import WorkflowNodeExecutionStatus -from graphon.graph import Graph from graphon.node_events import NodeRunResult from graphon.nodes.code.code_node import CodeNode from graphon.nodes.code.entities import CodeNodeData from graphon.nodes.code.limits import CodeNodeLimits from graphon.runtime import RuntimeState, VariablePool +from services.workflow.execution.adapters.node_factory import DefaultWorkflowCodeExecutor from tests.workflow_test_utils import build_test_graph_init_params pytest_plugins = ("tests.e2e_tests.middleware.workflow.nodes.__mock.code_executor",) @@ -47,14 +46,12 @@ def _init_code_node(code_config: dict) -> CodeNode: graph_runtime_state = RuntimeState( workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter() ) - node_factory = DifyNodeFactory(init_params=init_params, runtime_state=graph_runtime_state) - Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start") return CodeNode( node_id=str(uuid.uuid4()), data=CodeNodeData.model_validate(code_config["data"]), init_params=init_params, runtime_state=graph_runtime_state, - code_executor=node_factory._code_executor, + code_executor=DefaultWorkflowCodeExecutor(), code_limits=CodeNodeLimits( max_string_length=dify_config.CODE_MAX_STRING_LENGTH, max_number=dify_config.CODE_MAX_NUMBER, diff --git a/api/tests/e2e_tests/middleware/workflow/nodes/test_http.py b/api/tests/e2e_tests/middleware/workflow/nodes/test_http.py index 092f305506a259..adeeb26338b569 100644 --- a/api/tests/e2e_tests/middleware/workflow/nodes/test_http.py +++ b/api/tests/e2e_tests/middleware/workflow/nodes/test_http.py @@ -8,14 +8,12 @@ from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom from core.helper.ssrf_proxy import ssrf_proxy from core.tools.tool_file_manager import ToolFileManager -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import DifyFileReferenceFactory from core.workflow.system_variables import build_system_variables from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionStatus from graphon.file.file_manager import file_manager -from graphon.graph import Graph from graphon.nodes.http_request import HttpRequestNode, HttpRequestNodeConfig, HttpRequestNodeData from graphon.runtime import RuntimeState, VariablePool +from services.workflow.execution.adapters.node_runtime import DifyFileReferenceFactory from tests.workflow_test_utils import build_test_graph_init_params pytest_plugins = ("tests.e2e_tests.middleware.workflow.nodes.__mock.http",) @@ -68,14 +66,6 @@ def init_http_node(config: dict): workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter() ) - # Create node factory - node_factory = DifyNodeFactory( - init_params=init_params, - runtime_state=graph_runtime_state, - ) - - Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start") - node = HttpRequestNode( node_id=str(uuid.uuid4()), data=HttpRequestNodeData.model_validate(config["data"]), @@ -717,14 +707,6 @@ def test_nested_object_variable_selector(setup_http_mock): workflow_id="test-workflow", variable_pool=variable_pool, start_at=time.perf_counter() ) - # Create node factory - node_factory = DifyNodeFactory( - init_params=init_params, - runtime_state=graph_runtime_state, - ) - - Graph.init(graph_config=graph_config, node_factory=node_factory, root_node_id="start") - node = HttpRequestNode( node_id=str(uuid.uuid4()), data=HttpRequestNodeData.model_validate(graph_config["nodes"][1]["data"]), diff --git a/api/tests/helpers/unit/services/dataset_service_test_helpers.py b/api/tests/helpers/unit/services/dataset_service_test_helpers.py index 8864c9a046f9b5..70b5662127284f 100644 --- a/api/tests/helpers/unit/services/dataset_service_test_helpers.py +++ b/api/tests/helpers/unit/services/dataset_service_test_helpers.py @@ -32,26 +32,19 @@ ) from models.enums import DataSourceType from models.model import UploadFile +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from services.entities.knowledge_entities.rag_pipeline_entities import ( IconInfo as PipelineIconInfo, ) from services.entities.knowledge_entities.rag_pipeline_entities import ( - KnowledgeConfiguration, RagPipelineDatasetCreateEntity, ) -from services.entities.knowledge_entities.rag_pipeline_entities import ( - RerankingModelConfig as RagPipelineRerankingModelConfig, -) -from services.entities.knowledge_entities.rag_pipeline_entities import ( - RetrievalSetting as RagPipelineRetrievalSetting, -) from services.errors.base import NoPermissionError from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError from services.errors.dataset import DatasetNameDuplicateError from services.errors.document import DocumentIndexingError from services.errors.file import FileNotExistsError from services.knowledge.dataset_service import ( - DatasetCollectionBindingService, DatasetPermissionService, DatasetService, DocumentService, @@ -82,7 +75,7 @@ "CloudPlan", "DataSource", "Dataset", - "DatasetCollectionBindingService", + "DatasetCollectionBindingRepository", "DatasetNameDuplicateError", "DatasetPermissionEnum", "DatasetPermissionService", @@ -99,7 +92,6 @@ "IndexStructureType", "InfoList", "KnowledgeConfig", - "KnowledgeConfiguration", "LLMBadRequestError", "MagicMock", "Mock", @@ -115,8 +107,6 @@ "ProcessRule", "ProviderTokenNotInitError", "RagPipelineDatasetCreateEntity", - "RagPipelineRerankingModelConfig", - "RagPipelineRetrievalSetting", "RerankingModel", "RetrievalMethod", "RetrievalModel", @@ -131,7 +121,6 @@ "_make_dataset", "_make_document", "_make_features", - "_make_knowledge_configuration", "_make_lock_context", "_make_retrieval_model", "_make_segment", @@ -429,38 +418,3 @@ def _make_retrieval_model( top_k=4, score_threshold_enabled=False, ) - - -def _make_rag_pipeline_retrieval_setting() -> RagPipelineRetrievalSetting: - return RagPipelineRetrievalSetting( - search_method=RetrievalMethod.SEMANTIC_SEARCH, - top_k=4, - score_threshold=0.5, - score_threshold_enabled=True, - reranking_mode="reranking_model", - reranking_enable=True, - reranking_model=RagPipelineRerankingModelConfig( - reranking_provider_name="rerank-provider", - reranking_model_name="rerank-model", - ), - ) - - -def _make_knowledge_configuration( - *, - chunk_structure: str = "paragraph", - indexing_technique: str = "high_quality", - embedding_model_provider: str = "provider", - embedding_model: str = "embedding-model", - keyword_number: int = 8, - summary_index_setting: dict | None = None, -) -> KnowledgeConfiguration: - return KnowledgeConfiguration( - chunk_structure=chunk_structure, - indexing_technique=indexing_technique, - embedding_model_provider=embedding_model_provider, - embedding_model=embedding_model, - keyword_number=keyword_number, - retrieval_model=_make_rag_pipeline_retrieval_setting(), - summary_index_setting=summary_index_setting, - ) diff --git a/api/tests/unit_tests/commands/test_data_migration.py b/api/tests/unit_tests/commands/test_data_migration.py index 92f9df31884204..4bfe8b69f92e44 100644 --- a/api/tests/unit_tests/commands/test_data_migration.py +++ b/api/tests/unit_tests/commands/test_data_migration.py @@ -11,6 +11,7 @@ from pathlib import Path from types import ModuleType, SimpleNamespace from typing import cast, override +from unittest.mock import create_autospec import pytest import sqlalchemy as sa @@ -38,6 +39,7 @@ migration_data_wizard, parse_index_selection, ) +from extensions import ext_application_services from graphon.model_runtime.entities.model_entities import ModelType from models import Dataset, DatasetPermission, DatasetPermissionEnum, TenantAccountJoin, TenantAccountRole from models.account import Tenant @@ -55,6 +57,7 @@ ReportContext, ) from services.enterprise.rbac_service import ReplaceMemberBindings, ReplaceUserAccessPolicies +from services.workflow.console_service import ConsoleWorkflowService from tests.helpers.legacy_model_type_migration import ( ALL_TABLE_NAMES, LEGACY_TO_CANONICAL, @@ -137,7 +140,9 @@ def test_export_template_command_requires_overwrite_for_existing_output(tmp_path assert "already exists" in result.output @staticmethod - def test_export_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqlite_engine: Engine): + def test_export_command_uses_cli_owned_session( + monkeypatch, tmp_path: Path, sqlite_engine: Engine, workflow_application + ): captured: dict[str, object] = {} input_file = tmp_path / "export-config.json" output_file = tmp_path / "migration-package.json" @@ -145,6 +150,10 @@ def test_export_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqli package = MigrationPackage.from_mapping({"metadata": {"version": "1", "source_scope": "single"}}) class FakeMigrationExportService: + def __init__(self, *, workflow_tools, tool_providers): + assert tool_providers is workflow_application.tools.tool_providers + assert workflow_tools is workflow_application.tools.workflows + def export(self, selection, *, session): captured["session"] = session captured["selection"] = selection @@ -174,7 +183,9 @@ def save_package(self, package_to_save, path, *, overwrite): assert captured["overwrite"] is False @staticmethod - def test_import_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqlite_engine: Engine): + def test_import_command_uses_cli_owned_session( + monkeypatch, tmp_path: Path, sqlite_engine: Engine, workflow_application + ): captured: dict[str, object] = {} input_file = tmp_path / "migration-package.json" input_file.write_text("{}") @@ -189,7 +200,20 @@ def test_import_command_uses_cli_owned_session(monkeypatch, tmp_path: Path, sqli } ) + publisher = create_autospec(ConsoleWorkflowService, instance=True) + services = create_autospec(ext_application_services.ApplicationServices, instance=True) + services.console_workflows = publisher + services.tools = workflow_application.tools + monkeypatch.setattr(data_migration, "application_services", lambda: services) + class FakeMigrationImportService: + def __init__(self, *, workflows, app_dsl, workflow_tools): + assert workflow_tools is services.tools.workflows + from extensions.application_services.workflow import build_app_dsl_service + + assert workflows is publisher + assert app_dsl is build_app_dsl_service + def import_package(self, request, *, session): captured["session"] = session captured["request"] = request diff --git a/api/tests/unit_tests/conftest.py b/api/tests/unit_tests/conftest.py index ec3cbbde112037..52adf2bc04b113 100644 --- a/api/tests/unit_tests/conftest.py +++ b/api/tests/unit_tests/conftest.py @@ -29,11 +29,13 @@ if TYPE_CHECKING: from extensions.ext_application_services import ApplicationServices + from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.account_domain import AccountDomain # Getting the absolute path of the current file's directory ABS_PATH = os.path.dirname(os.path.abspath(__file__)) + # Getting the absolute path of the project's root directory PROJECT_DIR = os.path.abspath(os.path.join(ABS_PATH, os.pardir, os.pardir)) @@ -72,7 +74,6 @@ from services.tools.workflow_tools_manage_service import WorkflowToolManageService from services.workflow.console_variable_service import ConsoleWorkflowVariableService from services.workflow.generation.suggestions import WorkflowInstructionSuggestions - from services.workflow.variable_service import WorkflowVariableService def _patch_redis_clients_on_loaded_modules() -> None: @@ -362,10 +363,17 @@ def rbac_domain(sqlite_session_factory: sessionmaker[Session], monkeypatch: pyte @pytest.fixture -def app_services(sqlite_session_factory: sessionmaker[Session], rbac_domain: RBACDomain) -> AppServices: +def app_services( + sqlite_session_factory: sessionmaker[Session], + rbac_domain: RBACDomain, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, + workflow_variables: WorkflowExecutionVariables, +) -> AppServices: from unittest.mock import Mock from extensions.application_services.app import build_app_services + from extensions.application_services.workflow import build_console_workflow_service from extensions.ext_application_services import _build_oauth_server_service from extensions.ext_redis import redis_client from repositories.app.console_repository import ConsoleAppRepository @@ -384,6 +392,14 @@ def app_services(sqlite_session_factory: sessionmaker[Session], rbac_domain: RBA ), oauth=_build_oauth_server_service(database_client=sqlite_session_factory, redis=redis_client), recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()), + tool_providers=tool_providers, + workflow_tools=workflow_queries, + workflow_publisher=build_console_workflow_service( + database_client=sqlite_session_factory, + redis=redis_client, + variables=workflow_variables, + rbac_members=rbac_domain.rbac.members, + ), ) @@ -417,7 +433,7 @@ def workflow_tools(sqlite_session_factory: sessionmaker[Session]) -> WorkflowToo @pytest.fixture -def workflow_variables(sqlite_session_factory: sessionmaker[Session]) -> WorkflowVariableService: +def workflow_variables(sqlite_session_factory: sessionmaker[Session]) -> WorkflowExecutionVariables: from extensions.application_services.workflow_variables import build_workflow_variable_service return build_workflow_variable_service(database_client=sqlite_session_factory) diff --git a/api/tests/unit_tests/controllers/conftest.py b/api/tests/unit_tests/controllers/conftest.py index d150e267d39c07..c0bde88af60dcf 100644 --- a/api/tests/unit_tests/controllers/conftest.py +++ b/api/tests/unit_tests/controllers/conftest.py @@ -3,16 +3,14 @@ import sys from collections.abc import Iterator from dataclasses import dataclass -from functools import cache -from types import SimpleNamespace from unittest.mock import create_autospec import pytest from flask import Flask, Response, request_finished from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from extensions.application_services.app import AppServices +from extensions.application_services.tools import ToolServices, build_tool_services from extensions.ext_application_services import ( _batch_get_enterprise_webapp_access_modes, _batch_get_enterprise_webapp_user_permissions, @@ -25,10 +23,13 @@ from services.credentials.query import CredentialQuery from services.data_source.provider_service import DatasourceProviderService from services.enterprise.enterprise_service import EnterpriseService +from services.rag_pipeline.execution_service import RagPipelineExecutionService from services.tag_application_service import TagApplicationService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService from services.workflow.dsl_import import DslImportService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.controllers.error_contract import check_response, track_external_apis track_external_apis() @@ -62,17 +63,22 @@ def receiver(sender: Flask, response: Response, **_: object) -> None: @pytest.fixture(autouse=True) -def datasource_application_dependencies(monkeypatch: pytest.MonkeyPatch) -> None: - # Most controller tests never use these services. Build the mock on demand, - # with a fresh cache for each test to preserve mock isolation. - @cache - def application_services() -> SimpleNamespace: - return SimpleNamespace( - data_sources=SimpleNamespace( - providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True) - ) - ) - +def datasource_application_dependencies( + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], +) -> None: + registry = DatasourceApplicationStub( + tools=build_tool_services(sqlite_session_factory), + workflow_variables=workflow_variables, + knowledge=PipelineKnowledgeStub( + pipeline_generator=create_autospec(PipelineGenerator, instance=True, spec_set=True), + pipeline_execution=create_autospec(RagPipelineExecutionService, instance=True, spec_set=True), + ), + data_sources=PipelineDataSourceStub( + providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True) + ), + ) for name in ( "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow", "controllers.console.datasets.rag_pipeline.datasource_content_preview", @@ -80,12 +86,13 @@ def application_services() -> SimpleNamespace: ): module = sys.modules.get(name) if module is not None: - monkeypatch.setattr(module, "application_services", application_services) + monkeypatch.setattr(module, "application_services", lambda: registry) @dataclass(frozen=True) class PipelineKnowledgeStub: pipeline_generator: PipelineGenerator + pipeline_execution: RagPipelineExecutionService @dataclass(frozen=True) @@ -93,21 +100,40 @@ class PipelineDataSourceStub: providers: DatasourceProviderService +@dataclass(frozen=True) +class DatasourceApplicationStub: + tools: ToolServices + data_sources: PipelineDataSourceStub + workflow_variables: WorkflowExecutionVariables + knowledge: PipelineKnowledgeStub + + @dataclass(frozen=True) class PipelineApplicationStub: + tools: ToolServices + workflow_variables: WorkflowExecutionVariables knowledge: PipelineKnowledgeStub credential_queries: CredentialQuery data_sources: PipelineDataSourceStub @pytest.fixture -def pipeline_application(monkeypatch: pytest.MonkeyPatch) -> PipelineGenerator: +def pipeline_application( + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], +) -> PipelineGenerator: from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as console_workflow from controllers.service_api.dataset.rag_pipeline import rag_pipeline_workflow as service_api_workflow generator = create_autospec(PipelineGenerator, instance=True, spec_set=True) registry = PipelineApplicationStub( - knowledge=PipelineKnowledgeStub(pipeline_generator=generator), + tools=build_tool_services(sqlite_session_factory), + workflow_variables=workflow_variables, + knowledge=PipelineKnowledgeStub( + pipeline_generator=generator, + pipeline_execution=create_autospec(RagPipelineExecutionService, instance=True, spec_set=True), + ), credential_queries=create_autospec(CredentialQuery, instance=True, spec_set=True), data_sources=PipelineDataSourceStub( providers=create_autospec(DatasourceProviderService, instance=True, spec_set=True) diff --git a/api/tests/unit_tests/controllers/console/agent/test_roster.py b/api/tests/unit_tests/controllers/console/agent/test_roster.py index bdb327eb0709e7..d873522a53da17 100644 --- a/api/tests/unit_tests/controllers/console/agent/test_roster.py +++ b/api/tests/unit_tests/controllers/console/agent/test_roster.py @@ -1,4 +1,5 @@ from collections.abc import Generator +from dataclasses import replace from datetime import datetime from inspect import getsource, unwrap from types import SimpleNamespace @@ -8,8 +9,8 @@ import pytest from flask import Flask from sqlalchemy.orm import Session -from werkzeug.exceptions import Forbidden, InternalServerError, NotFound +from controllers.common.errors import AccessDeniedError, InternalServerError, NotFoundError from controllers.console import console_ns from controllers.console.agent import composer as composer_controller from controllers.console.agent import roster as roster_controller @@ -55,6 +56,7 @@ from controllers.console.app.completion import AgentBuildChatFinalizeApi, AgentChatMessageApi, AgentChatMessageStopApi from controllers.console.app.error import CompletionRequestError from enums import CloudPlan, DeploymentEdition +from extensions.ext_application_services import ApplicationServices from models.account import Account, TenantAccountRole from models.agent import Agent, AgentConfigDraftType, AgentScope, AgentSource, AgentStatus from models.enums import ApiTokenType, CustomizeTokenStrategy, TagType @@ -75,8 +77,13 @@ @pytest.fixture(autouse=True) -def rbac_services(monkeypatch: pytest.MonkeyPatch, rbac_domain: RBACDomain) -> None: - monkeypatch.setattr(roster_controller, "application_services", lambda: rbac_domain) +def rbac_services( + monkeypatch: pytest.MonkeyPatch, + rbac_domain: RBACDomain, + workflow_application: ApplicationServices, +) -> None: + services = replace(workflow_application, rbac=rbac_domain.rbac) + monkeypatch.setattr(roster_controller, "application_services", lambda: services) def _version_response(version_id: str = "version-1") -> dict[str, object]: @@ -263,6 +270,7 @@ def account_id() -> str: return "account-1" +@pytest.mark.usefixtures("workflow_application") def test_agent_app_list_and_create_use_agent_route( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, sqlite_session: Session ) -> None: @@ -550,6 +558,7 @@ def create_app(self, tenant_id: str, params: object, account: object, *, session assert create_params.agent_role == "" +@pytest.mark.usefixtures("workflow_application") def test_agent_app_detail_update_delete_resolve_app_from_agent_id( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, sqlite_session: Session ) -> None: @@ -960,6 +969,7 @@ def update_app_api_status(self, app_obj: object, enable_api: bool, *, session: o resolve_app.assert_called_once_with(unbound_session, tenant_id="tenant-1", agent_id=agent_id) +@pytest.mark.usefixtures("workflow_application") def test_agent_app_update_allows_empty_role( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: @@ -1155,7 +1165,7 @@ def test_agent_version_restore_requires_cloud_paid_plan( api = AgentRosterVersionRestoreApi() if not allowed: - with pytest.raises(Forbidden, match="This feature requires a paid plan.") as exc_info: + with pytest.raises(AccessDeniedError, match="This feature requires a paid plan.") as exc_info: unwrap(api.post)(api, session, "tenant-1", _account(), agent_id, version_id) assert exc_info.value.code == 403 restore.assert_not_called() @@ -1370,6 +1380,7 @@ def get_statistics_summary( def test_workflow_composer_get_put_validate_candidates_impact_and_save( app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str ) -> None: + monkeypatch.setattr(composer_controller, "application_services", lambda: Mock(workflow_variables=Mock())) app_model = _app_detail_obj(id="app-1") payload = { "variant": ComposerVariant.WORKFLOW.value, @@ -1552,7 +1563,11 @@ def test_workflow_impact_returns_empty_without_version(app: Flask) -> None: def test_agent_composer_routes_resolve_app_from_agent_id( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + *, + workflow_application: ApplicationServices, ) -> None: agent_id = "00000000-0000-0000-0000-000000000001" captured: dict[str, object] = {} @@ -1748,7 +1763,11 @@ def create_finalization_message(**kwargs: object) -> dict[str, object]: def test_build_chat_finalization_helper_forces_debug_build_and_push_prompt( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, unbound_session: Session + workflow_application: ApplicationServices, + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", tenant_id="tenant-1", mode=AppMode.AGENT) captured: dict[str, object] = {} @@ -1788,6 +1807,7 @@ def generate(**kwargs: object) -> object: generate_call = cast(dict[str, object], captured["generate"]) assert generate_call["app_model"] is app_model assert generate_call["streaming"] is True + assert generate_call["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], generate_call["args"]) assert args["draft_type"] == "debug_build" assert args["response_mode"] == "streaming" @@ -1850,6 +1870,7 @@ def test_drain_streaming_generate_response_raises_when_stream_ends_early() -> No ], ) def test_agent_chat_helper_resolves_scoped_conversation_and_forces_streaming( + workflow_application: ApplicationServices, app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, @@ -1890,6 +1911,7 @@ def resolve_debug_conversation(**kwargs: object) -> str: assert captured["app_model"] is app_model assert captured["user"] is current_user assert captured["streaming"] is True + assert captured["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], captured["args"]) assert args["response_mode"] == "streaming" assert args["conversation_id"] == "00000000-0000-0000-0000-000000000001" @@ -1901,7 +1923,11 @@ def resolve_debug_conversation(**kwargs: object) -> str: def test_agent_chat_helper_ignores_private_exit_intent_payload_key( - app: Flask, monkeypatch: pytest.MonkeyPatch, account_id: str, unbound_session: Session + workflow_application: ApplicationServices, + app: Flask, + monkeypatch: pytest.MonkeyPatch, + account_id: str, + unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", tenant_id="tenant-1", mode=AppMode.AGENT) current_user = _account(account_id=account_id) @@ -1939,6 +1965,7 @@ def generate(**kwargs: object) -> dict[str, object]: assert result == {"response": {"answer": "ok"}} assert captured["streaming"] is True + assert captured["variables"] is workflow_application.workflow_variables args = cast(dict[str, object], captured["args"]) assert args["response_mode"] == "streaming" assert args["conversation_id"] == "debug-conversation-1" @@ -1978,7 +2005,7 @@ def test_agent_chat_helper_rejects_foreign_debug_conversation_before_generation( **payload_extra, } ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): completion_controller._create_chat_message( current_tenant_id="tenant-1", current_user=_account(account_id=account_id), @@ -2099,7 +2126,7 @@ def get_app_backing_agent(self, **kwargs: object) -> object: @pytest.mark.parametrize( ("error", "expected"), [ - (completion_controller.services.errors.conversation.ConversationNotExistsError(), NotFound), + (completion_controller.services.errors.conversation.ConversationNotExistsError(), NotFoundError), ( completion_controller.services.errors.conversation.ConversationCompletedError(), completion_controller.ConversationCompletedError, @@ -2123,6 +2150,7 @@ def get_app_backing_agent(self, **kwargs: object) -> object: ], ) def test_agent_chat_helper_maps_generation_errors( + workflow_application: ApplicationServices, app: Flask, monkeypatch: pytest.MonkeyPatch, error: Exception, @@ -2130,12 +2158,15 @@ def test_agent_chat_helper_maps_generation_errors( unbound_session: Session, ) -> None: app_model = _app_detail_obj(id="app-1", mode=AppMode.CHAT) - monkeypatch.setattr(completion_controller.AppGenerateService, "generate", lambda **_: (_ for _ in ()).throw(error)) + generate = Mock(side_effect=error) + monkeypatch.setattr(completion_controller.AppGenerateService, "generate", generate) with app.test_request_context(json={"inputs": {}, "query": "hello"}): with pytest.raises(expected): completion_controller._create_chat_message( current_user=_account(), app_model=app_model, session=unbound_session ) + generate.assert_called_once() + assert generate.call_args.kwargs["variables"] is workflow_application.workflow_variables def test_dify_tool_candidate_response_keeps_granularity_fields() -> None: diff --git a/api/tests/unit_tests/controllers/console/app/test_app.py b/api/tests/unit_tests/controllers/console/app/test_app.py index 0c6ad2a8d8fd59..8492bebac346bf 100644 --- a/api/tests/unit_tests/controllers/console/app/test_app.py +++ b/api/tests/unit_tests/controllers/console/app/test_app.py @@ -25,6 +25,7 @@ from werkzeug.datastructures import MultiDict from werkzeug.exceptions import BadRequest, Forbidden, NotFound +from controllers.common.errors import NotFoundError from controllers.console import flask_admission from controllers.console.app import ( annotation as annotation_module, @@ -236,7 +237,7 @@ def test_completion_api_conversation_not_exists( with ( Session(sqlite_engine) as session, app.test_request_context("/", json={"inputs": {}, "model_config": {}, "query": "hi"}), - pytest.raises(NotFound), + pytest.raises(NotFoundError), ): method( api, @@ -1014,7 +1015,7 @@ def test_app_detail_with_site_includes_nested_serialization( @staticmethod def test_app_response_view_uses_the_caller_session_for_query_backed_fields( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers, workflow_queries ): app_obj = _persist_response_graph(sqlite_session) decoy_tenant_id = "00000000-0000-0000-0000-000000000002" @@ -1040,7 +1041,12 @@ def load_annotation_reply(session, app_id): monkeypatch.setattr("services.app_service.load_annotation_reply_config", load_annotation_reply) - view = AppResponseView(app_obj, session=sqlite_session) + view = AppResponseView( + app_obj, + session=sqlite_session, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ) site = view.site workflow = view.workflow model_config = view.app_model_config @@ -1629,3 +1635,6 @@ def test_rejects_agent_mode(self): def test_rejects_unknown_mode(self): with pytest.raises(ValidationError): CreateAppPayload.model_validate({"name": "X", "mode": "not-a-mode"}) + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/console/app/test_message.py b/api/tests/unit_tests/controllers/console/app/test_message.py index 48d04bb94c21c2..7817dfb86bb69e 100644 --- a/api/tests/unit_tests/controllers/console/app/test_message.py +++ b/api/tests/unit_tests/controllers/console/app/test_message.py @@ -1,16 +1,26 @@ from __future__ import annotations from datetime import UTC, datetime +from decimal import Decimal +from unittest.mock import MagicMock import pytest from flask import Flask from sqlalchemy.orm import Session +from controllers.common.errors import MessageFeedbackRatingRequiredError from controllers.console.app import message as message_module -from core.app.entities.app_invoke_entities import InvokeFrom from fields.conversation_fields import MessageResponseSource +from machinery.context import RequestContext from models.account import Account -from models.enums import ConversationFromSource, CreatorUserRole, FeedbackFromSource, FeedbackRating +from models.enums import ( + ConversationFromSource, + ConversationStatus, + CreatorUserRole, + FeedbackFromSource, + FeedbackRating, + InvokeFrom, +) from models.model import ( App, AppAnnotationHitHistory, @@ -21,6 +31,7 @@ MessageAnnotation, MessageFeedback, ) +from services.errors.message import FeedbackRatingRequiredError def _account() -> Account: @@ -55,7 +66,7 @@ def _persist_message(session: Session, *, message_id: str, app_id: str = "app-1" introduction="", system_instruction="", system_instruction_tokens=0, - status="normal", + status=ConversationStatus.NORMAL, invoke_from=InvokeFrom.DEBUGGER, from_source=ConversationFromSource.CONSOLE, from_end_user_id=None, @@ -69,14 +80,14 @@ def _persist_message(session: Session, *, message_id: str, app_id: str = "app-1" query="query", message="", message_tokens=0, - message_unit_price=0, - message_price_unit=0, + message_unit_price=Decimal(0), + message_price_unit=Decimal(0), answer="answer", answer_tokens=0, - answer_unit_price=0, - answer_price_unit=0, + answer_unit_price=Decimal(0), + answer_price_unit=Decimal(0), provider_response_latency=0, - total_price=0, + total_price=Decimal(0), currency="USD", invoke_from=InvokeFrom.DEBUGGER, from_source=ConversationFromSource.CONSOLE, @@ -141,13 +152,19 @@ def test_message_response_source_uses_caller_session_for_nested_fields(sqlite_se assert source.inputs == {"topic": "support"} assert source.user_feedback is feedback - assert source.feedbacks[0].from_account.id == "account-1" + feedback_account = source.feedbacks[0].from_account + assert feedback_account is not None + assert feedback_account.id == "account-1" annotation_source = source.annotation assert annotation_source is not None - assert annotation_source.account.id == "account-1" + annotation_account = annotation_source.account + assert annotation_account is not None + assert annotation_account.id == "account-1" annotation_hit_history_source = source.annotation_hit_history assert annotation_hit_history_source is not None - assert annotation_hit_history_source.annotation_create_account.id == "account-1" + annotation_create_account = annotation_hit_history_source.annotation_create_account + assert annotation_create_account is not None + assert annotation_create_account.id == "account-1" assert source.agent_thoughts == [thought] assert source.message_files == [] @@ -204,6 +221,30 @@ def test_message_feedback_payload_no_rating(app: Flask, monkeypatch: pytest.Monk assert payload.rating is None @staticmethod + def test_update_message_feedback_rejects_empty_rating_without_existing_feedback( + monkeypatch: pytest.MonkeyPatch, + ) -> None: + services = MagicMock() + services.message_feedbacks.set_admin_feedback.side_effect = FeedbackRatingRequiredError() + monkeypatch.setattr(message_module, "application_services", lambda: services) + message_id = "550e8400-e29b-41d4-a716-446655440000" + + with pytest.raises(MessageFeedbackRatingRequiredError): + message_module._update_message_feedback( + args=message_module.AppMessageFeedbackPayload(message_id=message_id), + context=RequestContext("request-1", None, "account-1", "tenant-1"), + app_id="app-1", + ) + + services.message_feedbacks.set_admin_feedback.assert_called_once_with( + app_id="app-1", + app_owner_tenant_id="tenant-1", + account_id="account-1", + message_id=message_id, + rating=None, + content=None, + ) + @staticmethod def test_message_detail_response_normalizes_aliases_and_timestamp( app: Flask, monkeypatch: pytest.MonkeyPatch diff --git a/api/tests/unit_tests/controllers/console/app/test_model_config.py b/api/tests/unit_tests/controllers/console/app/test_model_config.py index f12596824e731c..e366172bccde3e 100644 --- a/api/tests/unit_tests/controllers/console/app/test_model_config.py +++ b/api/tests/unit_tests/controllers/console/app/test_model_config.py @@ -859,6 +859,12 @@ def test_post_encrypts_agent_tool_parameters( }, }, ) + tool_services = SimpleNamespace(tool_providers=object(), workflow_queries=object()) + monkeypatch.setattr( + model_config_module, + "application_services", + lambda: SimpleNamespace(tools=tool_services), + ) monkeypatch.setattr(model_config_module.ToolManager, "get_agent_tool_runtime", lambda **_kwargs: object()) class _ParamManager: diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow.py b/api/tests/unit_tests/controllers/console/app/test_workflow.py index 355f74af1e0855..d8a82a0cf9c54b 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow.py @@ -1,1635 +1,314 @@ -from __future__ import annotations +"""Transport contracts; workflow behavior is tested at the use-case and adapter boundaries.""" import inspect -import json -from contextlib import contextmanager, nullcontext +from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime -from inspect import unwrap -from types import SimpleNamespace -from unittest.mock import ANY, MagicMock, Mock +from datetime import UTC, datetime +from unittest.mock import MagicMock, Mock, create_autospec +from uuid import UUID import pytest -from flask import Flask -from pydantic import ValidationError -from sqlalchemy.orm import Session -from werkzeug.exceptions import HTTPException, NotFound, UnprocessableEntity +from flask import Flask, Response +from flask_restx import Resource +from pydantic import BaseModel, JsonValue, ValidationError +from werkzeug.exceptions import HTTPException + +from controllers.common.errors import InvalidArgumentError, InvalidRequestError, NotFoundError +from controllers.console.app import workflow as controller +from controllers.console.app.error import DraftWorkflowNotSync +from graphon.variables.exc import VariableError +from libs.external_api import ExternalApi +from machinery.context import RequestContext +from services.agent.workflow_contracts import WorkflowAgentBindingStore +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError +from services.workflow.console_service import ConsoleWorkflowService +from services.workflow.contracts import WorkflowChange, WorkflowOwner, WorkflowPublication, WorkflowTriggerError + +CONTEXT = RequestContext("request-1", "trace-1", "account-1", "tenant-1") +APP_ID = UUID("00000000-0000-0000-0000-000000000001") -from controllers.common.errors import InvalidArgumentError -from controllers.console import wraps as console_wraps -from controllers.console.app import workflow as workflow_module -from controllers.console.app import wraps as app_wraps -from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync -from controllers.console.app.workflow import ConvertToWorkflowApi -from core.workflow.llm_environment_variable import LLMEnvironmentVariable -from enums import DeploymentEdition -from graphon.file import File, FileTransferMethod, FileType -from graphon.variables import SecretVariable, StringVariable -from graphon.variables.variables import RAGPipelineVariable -from libs import login as login_lib -from models import Account as _workflow_convert_api_Account -from models import App as _workflow_convert_api_App -from models import App as _workflow_human_input_debug_api_App -from models import Tenant -from models.account import Account as _workflow_Account -from models.account import Account as _workflow_human_input_debug_api_Account -from models.account import AccountStatus, TenantAccountRole -from models.model import App as _workflow_App -from models.model import AppMode, IconType -from models.workflow import Workflow, WorkflowType -from services import workflow_variable_reference_validator as variable_reference_validator -from services.app.access import AppAccessFilter -from tests.helpers.unit.config_override import apply_config_overrides -from tests.helpers.unit.controllers.error_body import raised_error -from tests.helpers.unit.model_factories import make_app - -def _account() -> _workflow_Account: - account = _workflow_Account(name="Alice", email="alice@example.com") - account.id = "user-1" - return account - - -def _workflow_app(*, app_id: str = "app", tenant_id: str = "t1") -> _workflow_App: - return _workflow_App( - id=app_id, - tenant_id=tenant_id, - name="Workflow App", - description="", - mode=AppMode.WORKFLOW, - enable_site=True, - enable_api=True, - max_active_requests=0, - ) - - -def _make_workflow(**overrides) -> Workflow: - graph = overrides.pop("graph_dict", {"nodes": [], "edges": []}) - features = overrides.pop("features_dict", {"file_upload": {"enabled": False}}) - environment_variables = overrides.pop( - "environment_variables", - [ - SecretVariable( - id="env-1", - name="API_KEY", - value="plain-token", - selector=["env", "API_KEY"], - description="API key", - ) - ], - ) - conversation_variables = overrides.pop( - "conversation_variables", - [ - StringVariable( - id="conv-1", - name="topic", - value="hello", - selector=["conversation", "topic"], - description="Topic", - ) - ], - ) - rag_pipeline_variables = overrides.pop( - "rag_pipeline_variables", - [ - RAGPipelineVariable.model_validate( - { - "variable": "query", - "type": "text-input", - "label": "Query", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": ["custom"], - "allowed_file_extensions": [".pdf"], - "allowed_file_upload_methods": ["local_file"], - } - ) - ], - ) - workflow = Workflow( - id="workflow-1", - tenant_id="t1", - app_id="app", - type=WorkflowType.WORKFLOW, - version="1", - graph=json.dumps(graph), - _features=json.dumps(features), - marked_name="Release 1", - marked_comment="Initial release", - created_by="user-1", - updated_by=None, - _environment_variables=[], - _conversation_variables=[], - _rag_pipeline_variables=[], - ) - workflow.created_at = datetime(2024, 1, 1, 12, 0, 0) - workflow.updated_at = datetime(2024, 1, 1, 12, 1, 0) - if environment_variables and isinstance(environment_variables[0], dict): - workflow._environment_variables = json.dumps( - {str(index): value for index, value in enumerate(environment_variables)} - ) - else: - workflow.environment_variables = environment_variables - workflow.conversation_variables = conversation_variables - workflow.rag_pipeline_variables = rag_pipeline_variables - for key, value in overrides.items(): - setattr(workflow, key, value) - return workflow - - -class TestWorkflowCases: - @staticmethod - @pytest.fixture(autouse=True) - def _identity_workflow_encryption(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep variable serialization focused on ORM behavior, not the external key provider.""" - - monkeypatch.setattr(workflow_module.encrypter, "encrypt_token", lambda *, tenant_id, token: token) - monkeypatch.setattr(workflow_module.encrypter, "decrypt_token", lambda *, tenant_id, token: token) - - @staticmethod - @pytest.mark.parametrize( - "advisory", ["clean", "warning", "checker-error", "formatter-error", "empty", "non-object", "invalid-json"] - ) - def test_publish_workflow_returns_success( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - advisory: str, - ) -> None: - current_user = SimpleNamespace(id="account-1") - app_model = SimpleNamespace(id="app-1", tenant_id="tenant-1") - graph = { - "nodes": [ - {"id": "start", "data": {"type": "start"}}, - {"id": "branch", "data": {"type": "if-else"}}, - {"id": "producer", "data": {"type": "code", "title": "Producer"}}, - { - "id": "consumer", - "data": {"type": "answer", "title": "Consumer", "answer": "{{#producer.text#}}"}, - }, - ], - "edges": [ - {"source": "start", "target": "branch"}, - {"source": "branch", "target": "producer", "sourceHandle": "true"}, - {"source": "branch", "target": "consumer", "sourceHandle": "false"}, - ], - } - workflow = SimpleNamespace( - id="published-workflow", - created_at=datetime(2026, 8, 17, 12, 0, 0), - graph={"clean": "{}", "empty": None, "non-object": "[]", "invalid-json": "{"}.get( - advisory, json.dumps(graph) - ), - ) - if advisory == "checker-error": - monkeypatch.setattr( - variable_reference_validator, "validate_variable_references", Mock(side_effect=RuntimeError("checker")) - ) - elif advisory == "formatter-error": - monkeypatch.setattr( - variable_reference_validator, - "format_variable_reference_errors", - Mock(side_effect=RuntimeError("format")), - ) - session = Mock() - session.get.return_value = app_model - monkeypatch.setattr( - workflow_module, - "WorkflowService", - Mock(return_value=SimpleNamespace(publish_workflow=Mock(return_value=workflow))), - ) - monkeypatch.setattr( - workflow_module, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=lambda: nullcontext(session)), - ) - monkeypatch.setattr(workflow_module, "db", SimpleNamespace(engine=object())) - with app.test_request_context("/apps/app-1/workflows/publish", method="POST", json={}): - response = inspect.unwrap(workflow_module.PublishedWorkflowApi.post)( - workflow_module.PublishedWorkflowApi(), - workflow_module.PublishWorkflowPayload.model_validate({}), - current_user, - app_model, - ) - - assert response["result"] == "success" - assert app_model.workflow_id == workflow.id - assert isinstance(response["created_at"], int) - if advisory == "warning": - assert "Consumer" in response["warning"] - assert "Producer" in response["warning"] - assert "skipped branch" in response["warning"] - else: - assert "warning" not in response - - @staticmethod - @pytest.mark.parametrize("transaction_fails", [False, True], ids=["commit-succeeds", "commit-fails"]) - def test_delete_workflow_retires_candidates_only_after_transaction_exit( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - transaction_fails: bool, - ) -> None: - current_user = SimpleNamespace(id="account-1") - app_model = SimpleNamespace(id="app-1", tenant_id="tenant-1") - session = Mock() - events: list[str] = [] - error = RuntimeError("commit failed") - workflow_service = SimpleNamespace( - delete_workflow=Mock(side_effect=lambda **_kwargs: events.append("delete") or ["inline-agent"]) - ) - - @contextmanager - def transaction(): - events.append("transaction-enter") - yield session - events.append("transaction-exit") - if transaction_fails: - raise error - - monkeypatch.setattr(workflow_module, "WorkflowService", Mock(return_value=workflow_service)) - monkeypatch.setattr( - workflow_module, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=transaction), - ) - monkeypatch.setattr(workflow_module, "db", SimpleNamespace(engine=object())) - retire_unowned = Mock(side_effect=lambda **_kwargs: events.append("retire")) - monkeypatch.setattr(workflow_module.WorkflowAgentRetirementService, "retire_unowned", retire_unowned) - - with app.test_request_context("/apps/app-1/workflows/workflow-1", method="DELETE"): - if transaction_fails: - with pytest.raises(RuntimeError) as exc_info: - inspect.unwrap(workflow_module.WorkflowByIdApi.delete)( - workflow_module.WorkflowByIdApi(), - current_user, - app_model, - "workflow-1", - ) - assert exc_info.value is error - else: - response = inspect.unwrap(workflow_module.WorkflowByIdApi.delete)( - workflow_module.WorkflowByIdApi(), - current_user, - app_model, - "workflow-1", - ) - assert response == (None, 204) - - assert events == ["transaction-enter", "delete", "transaction-exit"] + ([] if transaction_fails else ["retire"]) - if transaction_fails: - retire_unowned.assert_not_called() +@dataclass +class Services: + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + + +@pytest.fixture(name="workflows") +def workflow_use_cases(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + use_cases = create_autospec(ConsoleWorkflowService, instance=True, spec_set=True) + monkeypatch.setattr(controller, "application_services", lambda: Services(use_cases)) + return use_cases + + +def invoke( + resource: type[Resource], + handler: Callable[..., object], + *args: object, + **kwargs: object, +) -> object: + api = resource() + return inspect.unwrap(handler)(api, *args, request_context=CONTEXT, app_id=APP_ID, **kwargs) + + +@pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) +def test_sync_parses_transport_and_preserves_patch(app: Flask, workflows: Mock, content_type: str) -> None: + workflows.sync.return_value = WorkflowChange("new-hash", datetime(2024, 1, 1, tzinfo=UTC)) + body: dict[str, JsonValue] = { + "graph": {"nodes": []}, + "features": {}, + "hash": "old-hash", + "_is_collaborative": True, + "environment_variable_patch": { + "environment_variables": [{"id": "env-1", "name": "KEY", "value_type": "secret", "value": "masked"}], + "deleted_environment_variable_ids": ["env-2"], + }, + } + import json + + with app.test_request_context(method="POST", data=json.dumps(body), content_type=content_type): + response = invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) + assert response == {"result": "success", "hash": "new-hash", "updated_at": 1704067200} + context, app_id, command = workflows.sync.call_args.args + assert (context, app_id) == (CONTEXT, str(APP_ID)) + assert command.is_collaborative is True + assert command.unique_hash == "old-hash" + patch = body["environment_variable_patch"] + assert isinstance(patch, dict) + assert command.environment_upserts == patch["environment_variables"] + assert command.environment_deletions == ["env-2"] + + +@pytest.mark.parametrize( + ("content_type", "data", "status"), + [("application/xml", "x", 415), ("application/json", "[]", 400), ("text/plain", "invalid", 400)], +) +def test_sync_invalid_input_does_not_dispatch( + app: Flask, workflows: Mock, content_type: str, data: str, status: int +) -> None: + with app.test_request_context(method="POST", data=data, content_type=content_type): + if status == 415: + with pytest.raises(HTTPException) as error: + invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) + assert error.value.code == status else: - retire_unowned.assert_called_once_with( - tenant_id=app_model.tenant_id, - agent_ids=["inline-agent"], - account_id=current_user.id, - ) - - @staticmethod - def test_parse_file_no_config(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(workflow_module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: None) - workflow = _make_workflow(features_dict={}) - - assert workflow_module._parse_file(workflow, files=[{"id": "f"}]) == [] - - @staticmethod - def test_parse_file_with_config(monkeypatch: pytest.MonkeyPatch) -> None: - config = object() - file_list = [ - File( - tenant_id="t1", - file_type=FileType.IMAGE, - transfer_method=FileTransferMethod.REMOTE_URL, - remote_url="http://u", - ) - ] - build_mock = Mock(return_value=file_list) - monkeypatch.setattr(workflow_module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: config) - monkeypatch.setattr(workflow_module.file_factory, "build_from_mappings", build_mock) - - workflow = _make_workflow(features_dict={}) - result = workflow_module._parse_file(workflow, files=[{"id": "f"}]) - - assert result == file_list - build_mock.assert_called_once() - - @staticmethod - def test_sync_draft_workflow_invalid_content_type(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context("/apps/app/workflows/draft", method="POST", data="x", content_type="text/html"): - with pytest.raises(HTTPException) as exc: - handler(api, _account(), app_model=_workflow_app()) - - assert exc.value.code == 415 - - @staticmethod - def test_sync_draft_workflow_invalid_json(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data="[]", - content_type="application/json", - ): - response, status = raised_error(lambda: handler(api, _account(), app_model=_workflow_app())) - - assert status == 400 - assert response["message"] == "Invalid JSON data" - - @staticmethod - def test_sync_draft_workflow_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - - monkeypatch.setattr( - workflow_module.variable_factory, "build_environment_variable_from_mapping", lambda *_args: "env" - ) - monkeypatch.setattr( - workflow_module.variable_factory, "build_conversation_variable_from_mapping", lambda *_args: "conv" - ) - - sync_draft_workflow = Mock(return_value=workflow) - service = SimpleNamespace(sync_draft_workflow=sync_draft_workflow) - monkeypatch.setattr(workflow_module, "WorkflowService", lambda: service) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h"}, - ): - response = handler(api, _account(), app_model=_workflow_app()) - - assert response["result"] == "success" - assert sync_draft_workflow.call_args.kwargs["environment_variables"] == [] - assert sync_draft_workflow.call_args.kwargs["preserve_environment_variables"] is True - - @staticmethod - @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) - @pytest.mark.parametrize("confirmation", [{}, {"force": False}], ids=["missing-force", "force-false"]) - def test_sync_draft_workflow_rejects_empty_graph_without_confirmation( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - content_type: str, - confirmation: dict[str, bool], - ) -> None: - workflow_service = Mock() - monkeypatch.setattr(workflow_module, "WorkflowService", workflow_service) - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data=json.dumps({"graph": {"nodes": [], "edges": []}, "features": {}, "hash": "h", **confirmation}), - content_type=content_type, - ): - with pytest.raises(InvalidArgumentError) as exc: - handler(api, _account(), app_model=_workflow_app()) - - assert exc.value.code == 400 - assert exc.value.description == "Saving an empty workflow requires force=true." - workflow_service.assert_not_called() - - @staticmethod - @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) - @pytest.mark.parametrize( - ("graph", "confirmation"), - [ - ({"nodes": [], "edges": []}, {"force": True}), - ({"nodes": [{"id": "start"}], "edges": []}, {}), - ({"nodes": [], "edges": [{"id": "edge-1", "source": "start", "target": "end"}]}, {}), - ], - ids=["confirmed-empty-graph", "nodes-without-edges", "edges-without-nodes"], + with pytest.raises(InvalidArgumentError, match="Invalid JSON data"): + invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) + workflows.sync.assert_not_called() + + +@pytest.mark.parametrize( + ("failure", "error_type"), + [(WorkflowHashNotEqualError(), DraftWorkflowNotSync), (VariableError("bad variable"), InvalidArgumentError)], +) +def test_sync_domain_errors(app: Flask, workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: + workflows.sync.side_effect = failure + with app.test_request_context(method="POST", json={"graph": {}, "features": {}}), pytest.raises(error_type): + invoke(controller.DraftWorkflowApi, controller.DraftWorkflowApi.post) + + +@pytest.mark.parametrize( + "patch", + [ + {"environment_variables": [{"id": "same"}, {"id": "same"}]}, + {"environment_variables": [{"name": "missing-id"}]}, + {"environment_variables": [{"id": "same"}], "deleted_environment_variable_ids": ["same"]}, + {"deleted_environment_variable_ids": [""]}, + ], +) +def test_sync_rejects_ambiguous_variable_patch(patch: dict[str, object]) -> None: + with pytest.raises(ValidationError): + controller.SyncDraftWorkflowPayload.model_validate( + {"graph": {}, "features": {}, "environment_variable_patch": patch} + ) + + +def test_sync_rejects_removed_full_environment_payload() -> None: + with pytest.raises(ValidationError): + controller.SyncDraftWorkflowPayload.model_validate({"graph": {}, "features": {}, "environment_variables": []}) + + +@pytest.mark.parametrize("warning", [None, "branch may be skipped"]) +def test_publish_serializes_advisory(workflows: Mock, warning: str | None) -> None: + workflows.publish.return_value = (WorkflowPublication(datetime(2024, 1, 1, tzinfo=UTC), "{}"), warning) + response = invoke( + controller.PublishedWorkflowApi, + controller.PublishedWorkflowApi.post, + controller.PublishWorkflowPayload(), ) - def test_sync_draft_workflow_saves_confirmed_empty_or_nonempty_graph( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - content_type: str, - graph: dict[str, list[dict[str, str]]], - confirmation: dict[str, bool], - ) -> None: - workflow = SimpleNamespace(unique_hash="next-hash", updated_at=None, created_at=datetime(2024, 1, 1)) - sync_draft_workflow = Mock(return_value=workflow) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(sync_draft_workflow=sync_draft_workflow), - ) - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data=json.dumps({"graph": graph, "features": {}, "hash": "current-hash", **confirmation}), - content_type=content_type, - ): - response = handler(api, _account(), app_model=_workflow_app()) - - assert response["result"] == "success" - assert response["hash"] == "next-hash" - sync_draft_workflow.assert_called_once() - assert sync_draft_workflow.call_args.kwargs["graph"] == graph - assert sync_draft_workflow.call_args.kwargs["unique_hash"] == "current-hash" - - @staticmethod - @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) - def test_sync_draft_workflow_force_does_not_bypass_hash_conflict( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - content_type: str, - ) -> None: - sync_draft_workflow = Mock(side_effect=workflow_module.WorkflowHashNotEqualError()) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(sync_draft_workflow=sync_draft_workflow), - ) - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data=json.dumps({"graph": {"nodes": [], "edges": []}, "features": {}, "hash": "stale-hash", "force": True}), - content_type=content_type, - ): - with pytest.raises(DraftWorkflowNotSync) as exc: - handler(api, _account(), app_model=_workflow_app()) - - assert exc.value.code == 409 - sync_draft_workflow.assert_called_once() - assert sync_draft_workflow.call_args.kwargs["unique_hash"] == "stale-hash" - - @staticmethod - @pytest.mark.parametrize("content_type", ["application/json", "text/plain"]) - @pytest.mark.parametrize("force", ["true", 1, None], ids=["string", "integer", "null"]) - def test_sync_draft_workflow_rejects_non_boolean_force( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - content_type: str, - force: str | int | None, - ) -> None: - workflow_service = Mock() - monkeypatch.setattr(workflow_module, "WorkflowService", workflow_service) - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - data=json.dumps({"graph": {"nodes": [], "edges": []}, "features": {}, "force": force}), - content_type=content_type, - ): - if content_type == "application/json": - with pytest.raises(ValidationError, match="Input should be a valid boolean"): - handler(api, _account(), app_model=_workflow_app()) - else: - response, status = raised_error(lambda: handler(api, _account(), app_model=_workflow_app())) - assert status == 400 - assert response["message"] == "Invalid JSON data" - - workflow_service.assert_not_called() - - @staticmethod - def test_sync_draft_workflow_passes_environment_patch(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - patched_variable = StringVariable( - id="env-model", - name="shared_model", - value="model", - selector=["env", "shared_model"], - ) - build_environment_variable = Mock(return_value=patched_variable) - sync_draft_workflow = Mock(return_value=workflow) - monkeypatch.setattr( - workflow_module.variable_factory, - "build_environment_variable_from_mapping", - build_environment_variable, - ) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(sync_draft_workflow=sync_draft_workflow), - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={ - "graph": {}, - "features": {}, - "hash": "current-hash", - "environment_variable_patch": { - "environment_variables": [ - { - "id": "env-model", - "name": "shared_model", - "value": "model", - "value_type": "string", - } - ], - "deleted_environment_variable_ids": ["env-old"], - }, - }, - ): - response = handler(api, _account(), app_model=_workflow_app()) - - assert response["result"] == "success" - assert sync_draft_workflow.call_args.kwargs["preserve_environment_variables"] is True - assert sync_draft_workflow.call_args.kwargs["environment_variable_upserts"] == [patched_variable] - assert sync_draft_workflow.call_args.kwargs["deleted_environment_variable_ids"] == ["env-old"] - build_environment_variable.assert_called_once() - - @staticmethod - def test_sync_draft_workflow_rejects_overlapping_environment_patch_ids() -> None: - with pytest.raises(ValidationError, match="cannot be upserted and deleted"): - workflow_module.SyncDraftWorkflowPayload.model_validate( - { - "graph": {}, - "features": {}, - "environment_variable_patch": { - "environment_variables": [{"id": "env-model"}], - "deleted_environment_variable_ids": ["env-model"], - }, - } + assert isinstance(response, dict) + assert response["created_at"] == 1704067200 + assert response.get("warning") == warning + assert ("warning" in response) is (warning is not None) + workflows.publish.assert_called_once_with(CONTEXT, str(APP_ID), marked_name="", marked_comment="") + + +@pytest.mark.parametrize( + ("failure", "error_type"), + [ + (IsDraftWorkflowError(), InvalidRequestError), + (WorkflowNotFoundError("missing"), NotFoundError), + (ValueError("invalid graph"), InvalidRequestError), + ], +) +def test_restore_error_contract(workflows: Mock, failure: Exception, error_type: type[Exception]) -> None: + workflows.restore.side_effect = failure + with pytest.raises(error_type): + invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", + ) + + +@pytest.mark.parametrize( + ("failure", "status", "code"), + [(WorkflowNotFoundError("missing"), 404, "not_found"), (ValueError("invalid graph"), 400, "bad_request")], +) +def test_restore_domain_error_http_response(workflows: Mock, failure: Exception, status: int, code: str) -> None: + from controllers.console import api as console_api + + workflows.restore.side_effect = failure + app = Flask(__name__) + app.config["RESTX_ERROR_404_HELP"] = False + api = ExternalApi(app) + api.error_handlers = console_api.error_handlers.copy() + + class Restore(Resource): + def post(self) -> object: + return invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", ) - @staticmethod - def test_sync_draft_workflow_rejects_legacy_environment_variables() -> None: - with pytest.raises(ValidationError, match="Extra inputs are not permitted"): - workflow_module.SyncDraftWorkflowPayload.model_validate( - { - "graph": {}, - "features": {}, - "environment_variables": [], - } - ) - - @staticmethod - def test_sync_draft_workflow_hash_mismatch(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - def _raise(*_args, **_kwargs): - raise workflow_module.WorkflowHashNotEqualError() - - service = SimpleNamespace(sync_draft_workflow=_raise) - monkeypatch.setattr(workflow_module, "WorkflowService", lambda: service) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h"}, - ): - with pytest.raises(DraftWorkflowNotSync): - handler(api, _account(), app_model=_workflow_app()) - - @staticmethod - def test_sync_draft_workflow_variable_validation_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - def _raise(*_args, **_kwargs): - raise workflow_module.VariableError("description too long") - - monkeypatch.setattr(workflow_module.variable_factory, "build_conversation_variable_from_mapping", _raise) - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(sync_draft_workflow=lambda **_kwargs: None) - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft", - method="POST", - json={"graph": {}, "features": {}, "hash": "h", "conversation_variables": [{"name": "topic"}]}, - ): - with pytest.raises(InvalidArgumentError) as exc: - handler(api, _account(), app_model=_workflow_app()) - - assert exc.value.description == "description too long" - - @staticmethod - def test_restore_published_workflow_to_draft_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - workflow = _make_workflow(updated_at=None, created_at=datetime(2024, 1, 1)) - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(restore_published_workflow_to_draft=lambda **_kwargs: workflow), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - response = handler( - api, - _account(), - app_model=_workflow_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - assert response["result"] == "success" - assert response["hash"] == workflow.unique_hash - - @staticmethod - def test_restore_published_workflow_to_draft_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - workflow_module.WorkflowNotFoundError("Workflow not found") - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - with pytest.raises(NotFound): - handler( - api, - _account(), - app_model=_workflow_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - @staticmethod - def test_restore_published_workflow_to_draft_returns_400_for_draft_source( - app: Flask, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - workflow_module.IsDraftWorkflowError( - "Cannot use draft workflow version. Workflow ID: draft-workflow. " - "Please use a published workflow version or leave workflow_id empty." - ) - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/draft-workflow/restore", - method="POST", - ): - with pytest.raises(HTTPException) as exc: - handler( - api, - _account(), - app_model=_workflow_app(tenant_id="tenant-1"), - workflow_id="draft-workflow", - ) - - assert exc.value.code == 400 - assert exc.value.description == workflow_module.RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE - - @staticmethod - def test_restore_published_workflow_to_draft_returns_400_for_invalid_structure( - app: Flask, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - restore_published_workflow_to_draft=lambda **_kwargs: (_ for _ in ()).throw( - ValueError("invalid workflow graph") - ) - ), - ) - - api = workflow_module.DraftWorkflowRestoreApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/workflows/published-workflow/restore", - method="POST", - ): - with pytest.raises(HTTPException) as exc: - handler( - api, - _account(), - app_model=_workflow_app(tenant_id="tenant-1"), - workflow_id="published-workflow", - ) - - assert exc.value.code == 400 - assert exc.value.description == "invalid workflow graph" - - @staticmethod - def test_get_published_workflows_uses_the_request_session( - app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ) -> None: - api = workflow_module.PublishedAllWorkflowApi() - handler = inspect.unwrap(api.get) - workflow = _make_workflow() - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - def get_all_published_workflow(*, session: Session, **_kwargs): - assert session is sqlite_session - return [workflow], False - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_all_published_workflow=get_all_published_workflow), - ) - - with app.test_request_context( - "/apps/app/workflows", - method="GET", - query_string={"page": 1, "limit": 10, "user_id": "", "named_only": "false"}, - ): - query = workflow_module.WorkflowListQuery.model_validate( - {"page": "1", "limit": "10", "user_id": "", "named_only": "false"} - ) - response = handler(api, query, sqlite_session, _account(), app_model=_workflow_app()) - - assert response["items"][0]["id"] == "workflow-1" - assert response["page"] == 1 - assert response["limit"] == 10 - assert response["has_more"] is False - - @staticmethod - def test_draft_workflow_get_serializes_response_model( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ) -> None: - workflow = _make_workflow() - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(get_draft_workflow=lambda **_kwargs: workflow) - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - - response = handler(api, sqlite_session, app_model=_workflow_app()) - - assert response["id"] == "workflow-1" - assert response["graph"] == {"nodes": [], "edges": []} - assert response["features"] == {"file_upload": {"enabled": False}} - assert response["hash"] == workflow.unique_hash - assert response["created_by"] == {"id": "user-1", "name": "Alice", "email": "alice@example.com"} - assert response["updated_by"] is None - assert response["created_at"] == int(datetime(2024, 1, 1, 12, 0, 0).timestamp()) - assert response["updated_at"] == int(datetime(2024, 1, 1, 12, 1, 0).timestamp()) - assert response["environment_variables"] == [ - { - "id": "env-1", - "name": "API_KEY", - "value": workflow_module.encrypter.full_mask_token(), - "value_type": "secret", - "description": "API key", - } - ] - assert response["conversation_variables"] == [ - { - "id": "conv-1", - "name": "topic", - "value": "hello", - "value_type": "string", - "description": "Topic", - } - ] - assert response["rag_pipeline_variables"] == [ - { - "label": "Query", - "variable": "query", - "type": "text-input", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": ["custom"], - "allowed_file_extensions": [".pdf"], - "allowed_file_upload_methods": ["local_file"], - } - ] - - @staticmethod - def test_published_workflow_get_uses_session_aware_response_source( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ) -> None: - sqlite_session.add(_account()) - sqlite_session.commit() - workflow = _make_workflow() - monkeypatch.setattr(workflow_module.db, "session", lambda: sqlite_session) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_published_workflow=lambda **_kwargs: workflow), - ) - - api = workflow_module.PublishedWorkflowApi() - handler = inspect.unwrap(api.get) - - response = handler(api, app_model=SimpleNamespace(id="app")) - - assert response["id"] == "workflow-1" - assert response["created_by"] == {"id": "user-1", "name": "Alice", "email": "alice@example.com"} - assert response["updated_by"] is None - assert response["tool_published"] is False - - @staticmethod - def test_pipeline_variable_response_accepts_legacy_file_field_names() -> None: - response = workflow_module.PipelineVariableResponse.model_validate( - { - "label": "Query", - "variable": "query", - "type": "single-file", - "belong_to_node_id": "shared", - "max_length": 0, - "required": False, - "unit": "", - "default_value": "", - "options": [], - "placeholder": "", - "tooltips": "", - "allowed_file_types": [], - "allow_file_extension": [".txt"], - "allow_file_upload_methods": ["remote_url"], - } - ).model_dump(mode="json") - - assert response["allowed_file_extensions"] == [".txt"] - assert response["allowed_file_upload_methods"] == ["remote_url"] - - @staticmethod - def test_pipeline_variable_response_accepts_explicit_null_optional_fields() -> None: - pipeline_variable = RAGPipelineVariable.model_validate( - { - "label": "Query", - "variable": "query", - "type": "text-input", - "belong_to_node_id": "shared", - "max_length": None, - "unit": None, - "default_value": None, - "options": None, - "placeholder": None, - "tooltips": None, - "allowed_file_types": None, - "allowed_file_extensions": None, - "allowed_file_upload_methods": None, - } - ).model_dump(mode="json") - - response = workflow_module.PipelineVariableResponse.model_validate(pipeline_variable).model_dump(mode="json") - - assert response["max_length"] is None - assert response["allowed_file_types"] is None - assert response["allowed_file_extensions"] is None - assert response["allowed_file_upload_methods"] is None - - @staticmethod - def test_workflow_response_masks_secret_environment_variables(sqlite_session: Session) -> None: - workflow = _make_workflow( - environment_variables=[ - SecretVariable(id="env-secret", name="API_KEY", value="plain-token", selector=["env", "API_KEY"]), - StringVariable(id="env-string", name="REGION", value="us-east-1", selector=["env", "REGION"]), - ] - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - response = workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ).model_dump(mode="json") - - assert response["environment_variables"] == [ - { - "id": "env-secret", - "name": "API_KEY", - "value": workflow_module.encrypter.full_mask_token(), - "value_type": "secret", - "description": "", - }, - { - "id": "env-string", - "name": "REGION", - "value": "us-east-1", - "value_type": "string", - "description": "", - }, - ] - - @staticmethod - def test_workflow_response_preserves_llm_environment_variable_type(sqlite_session: Session) -> None: - workflow = _make_workflow( - environment_variables=[ - LLMEnvironmentVariable( - id="env-llm", - name="for_summarize", - value={"provider": "provider", "name": "model", "mode": "chat"}, - selector=["env", "for_summarize"], - ) - ] - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - response = workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ).model_dump(mode="json") - - assert response["environment_variables"] == [ - { - "id": "env-llm", - "name": "for_summarize", - "value": {"provider": "provider", "name": "model", "mode": "chat"}, - "value_type": "llm", - "description": "", - } - ] - - @staticmethod - def test_workflow_response_rejects_invalid_environment_variable_dict(sqlite_session: Session) -> None: - workflow = _make_workflow(environment_variables=[{"value_type": "not-a-segment-type"}]) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - - with pytest.raises(ValidationError): - workflow_module.WorkflowResponse.model_validate( - workflow_module.WorkflowResponseSource(workflow, session=sqlite_session), - from_attributes=True, - ) - - @staticmethod - def test_draft_workflow_get_not_found(monkeypatch: pytest.MonkeyPatch, unbound_session: Session) -> None: - monkeypatch.setattr( - workflow_module, "WorkflowService", lambda: SimpleNamespace(get_draft_workflow=lambda **_k: None) - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - - with pytest.raises(DraftWorkflowNotExist): - handler(api, unbound_session, app_model=_workflow_app()) - - @staticmethod - def test_draft_workflow_get_projects_agent_node_job_to_graph( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session - ) -> None: - workflow = _make_workflow( - graph_dict={ - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - }, - } - ], - "edges": [], - } - ) - sqlite_session.add_all([_account(), workflow]) - sqlite_session.commit() - projected_graph = { - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - "agent_task": "Summarize it.", - "agent_declared_outputs": [{"name": "summary", "type": "string"}], - }, - } - ], - "edges": [], - } - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_draft_workflow=lambda **_k: workflow), - ) - - from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService - - monkeypatch.setattr( - WorkflowAgentPublishService, - "project_draft_bindings_to_graph", - lambda **_k: projected_graph, - ) - - api = workflow_module.DraftWorkflowApi() - handler = inspect.unwrap(api.get) - - response = handler(api, sqlite_session, app_model=_workflow_app()) - - assert response["graph"] == projected_graph - - @staticmethod - def test_advanced_chat_run_conversation_not_exists( - app: Flask, monkeypatch: pytest.MonkeyPatch, unbound_session: Session - ) -> None: - monkeypatch.setattr( - workflow_module.AppGenerateService, - "generate", - lambda *_args, **_kwargs: (_ for _ in ()).throw( - workflow_module.services.errors.conversation.ConversationNotExistsError() - ), - ) - - api = workflow_module.AdvancedChatDraftWorkflowRunApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/app/advanced-chat/workflows/draft/run", - method="POST", - json={"inputs": {}}, - ): - payload = workflow_module.AdvancedChatWorkflowRunPayload.model_validate({"inputs": {}}) - with pytest.raises(NotFound): - handler(api, payload, unbound_session, _account(), app_model=_workflow_app()) - - @staticmethod - @pytest.mark.parametrize( - ("resource", "payload_model", "payload"), - [ - ( - workflow_module.DraftWorkflowTriggerRunApi, - workflow_module.DraftWorkflowTriggerRunPayload, - {"node_id": "node-1"}, - ), - ( - workflow_module.DraftWorkflowTriggerRunAllApi, - workflow_module.DraftWorkflowTriggerRunAllPayload, - {"node_ids": ["node-1"]}, - ), - ], - ) - def test_trigger_run_loads_draft_with_request_session( - app: Flask, - monkeypatch: pytest.MonkeyPatch, - unbound_session: Session, - resource: type, - payload_model: type, - payload: dict[str, object], - ) -> None: - get_draft_workflow = Mock(return_value=None) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_draft_workflow=get_draft_workflow), - ) - session = unbound_session - app_model = _workflow_app(app_id="app-1") - handler = inspect.unwrap(resource.post) - - with app.test_request_context("/", method="POST", json=payload): - with pytest.raises(ValueError, match="Workflow not found"): - handler( - resource(), - payload_model.model_validate(payload), - session, - _account(), - app_model, - ) - - get_draft_workflow.assert_called_once_with(app_model, session=session) - - @staticmethod - def test_workflow_online_users_filters_inaccessible_workflow( - app: Flask, monkeypatch: pytest.MonkeyPatch, unbound_session: Session - ) -> None: - app_id_1 = "11111111-1111-1111-1111-111111111111" - app_id_2 = "22222222-2222-2222-2222-222222222222" - signed_avatar_url = "https://files.example.com/signed/avatar-1" - signed_ids: list[str] = [] - queries: list[tuple[list[str], str, Session]] = [] - actors: list[tuple[str, str]] = [] - - def sign_avatar(file_id: str) -> str: - signed_ids.append(file_id) - return signed_avatar_url - - class WorkflowQueries: - def get_tenant_app_maintainers( - self, app_ids: list[str], tenant_id: str, *, session: Session - ) -> dict[str, str]: - queries.append((app_ids, tenant_id, session)) - return {app_id_1: "owner-1", app_id_2: "owner-2"} - - def resolve_access(tenant_id: str, account_id: str) -> AppAccessFilter: - actors.append((tenant_id, account_id)) - return AppAccessFilter(accessible_app_ids={app_id_1}) - - monkeypatch.setattr(workflow_module, "WorkflowService", WorkflowQueries) - monkeypatch.setattr(workflow_module, "resolve_app_access_filter", resolve_access) - apply_config_overrides(monkeypatch, RBAC_ENABLED=True) - monkeypatch.setattr(workflow_module.file_helpers, "get_signed_file_url", sign_avatar) - monkeypatch.setattr(workflow_module.session_factory, "create_session", lambda: nullcontext(unbound_session)) - - redis_result = [ - { - b"sid-1": json.dumps( - { - "user_id": "u-1", - "username": "Alice", - "avatar": "avatar-file-id", - "sid": "sid-1", - } - ), - b"sid-malformed": json.dumps({"avatar": "avatar-file-id", "sid": "sid-malformed"}), - b"sid-invalid-avatar": json.dumps( - { - "user_id": "u-2", - "username": "Bob", - "avatar": {"file_id": "avatar-file-id"}, - } - ), - b"sid-invalid-user-id": json.dumps( - { - "user_id": 42, - "username": "Carol", - "avatar": "avatar-file-id", - } - ), - b"sid-invalid-username": json.dumps( - { - "user_id": "u-4", - "username": ["Dave"], - "avatar": "avatar-file-id", - } - ), - } - ] - redis_keys: list[str] = [] - pipeline_transactions: list[bool] = [] - executions: list[bool] = [] - - class PresencePipeline: - def hgetall(self, key: str) -> None: - redis_keys.append(key) - - def execute(self) -> list[dict[bytes, str]]: - executions.append(True) - return redis_result - - def pipeline(*, transaction: bool) -> PresencePipeline: - pipeline_transactions.append(transaction) - return PresencePipeline() - - monkeypatch.setattr(workflow_module.redis_client, "pipeline", pipeline) - - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": [app_id_1, app_id_2]}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": [app_id_1, app_id_2]}) - actor = _account() - actor.id = "account-1" - response = handler(api, args, "tenant-1", actor) - - assert response == { - "data": [ - { - "app_id": app_id_1, - "users": [ - { - "user_id": "u-1", - "username": "Alice", - "avatar": signed_avatar_url, - }, - { - "user_id": "u-2", - "username": "Bob", - "avatar": None, - }, - ], - } - ] - } - assert pipeline_transactions == [False] - assert redis_keys == [f"{workflow_module.WORKFLOW_ONLINE_USERS_PREFIX}{app_id_1}"] - assert executions == [True] - assert signed_ids == ["avatar-file-id"] - assert queries == [([app_id_1, app_id_2], "tenant-1", unbound_session)] - assert actors == [("tenant-1", "account-1")] - - @staticmethod - def test_workflow_online_users_batches_redis_reads(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - app_ids = [f"wf-{index}" for index in range(workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE + 1)] - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace( - get_tenant_app_maintainers=lambda app_ids, tenant_id, session: dict.fromkeys(app_ids) - ), - ) - apply_config_overrides(monkeypatch, RBAC_ENABLED=False) - monkeypatch.setattr(workflow_module.session_factory, "create_session", lambda: nullcontext(Mock())) - - first_pipeline = Mock() - first_pipeline.execute.return_value = [ - {} for _ in range(workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE) - ] - second_pipeline = Mock() - second_pipeline.execute.return_value = [{}] - redis_pipeline_factory = Mock(side_effect=[first_pipeline, second_pipeline]) - monkeypatch.setattr(workflow_module.redis_client, "pipeline", redis_pipeline_factory) - - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": app_ids}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": app_ids}) - response = handler(api, args, "tenant-1", SimpleNamespace(id="account-1")) - - assert len(response["data"]) == len(app_ids) - assert redis_pipeline_factory.call_count == 2 - assert first_pipeline.hgetall.call_count == workflow_module.WORKFLOW_ONLINE_USERS_REDIS_BATCH_SIZE - assert second_pipeline.hgetall.call_count == 1 - - @staticmethod - def test_workflow_online_users_rejects_excessive_workflow_ids(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - get_tenant_app_maintainers = Mock(return_value={}) - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: SimpleNamespace(get_tenant_app_maintainers=get_tenant_app_maintainers), - ) - - excessive_ids = [f"wf-{index}" for index in range(workflow_module.MAX_WORKFLOW_ONLINE_USERS_REQUEST_IDS + 1)] - - api = workflow_module.WorkflowOnlineUsersApi() - handler = inspect.unwrap(api.post) - - with app.test_request_context( - "/apps/workflows/online-users", - method="POST", - json={"app_ids": excessive_ids}, - ): - args = workflow_module.WorkflowOnlineUsersPayload.model_validate({"app_ids": excessive_ids}) - with pytest.raises(HTTPException) as exc: - handler(api, args, "tenant-1", SimpleNamespace(id="account-1")) - - assert exc.value.code == 400 - assert exc.value.description is not None - assert "Maximum" in exc.value.description - get_tenant_app_maintainers.assert_not_called() - - -"""Unit tests for convert-to-workflow endpoint.""" - - -def _workflow_convert_api_app(app_id: str) -> _workflow_convert_api_App: - return make_app(app_id=app_id, name=f"App {app_id}") - - -class TestWorkflowConvertApiCases: - class TestConvertToWorkflowApi: - @pytest.fixture - def api(self): - return workflow_module.ConvertToWorkflowApi() - - def test_convert_to_workflow_attaches_permission_keys_when_rbac_enabled( - self, api: ConvertToWorkflowApi, app: Flask, monkeypatch: pytest.MonkeyPatch - ) -> None: - method = unwrap(api.post) - new_app = _workflow_convert_api_app("new-app-1") - - monkeypatch.setattr( - workflow_module, - "WorkflowService", - lambda: type("WorkflowServiceStub", (), {"convert_to_workflow": lambda self, **_kwargs: new_app})(), - ) - monkeypatch.setattr( - workflow_module, - "get_app_permission_keys", - lambda tenant_id, account_id, app_id: ["app.acl.view_layout", "app.acl.edit"], - ) - - with app.test_request_context( - "/console/api/apps/app-1/convert-to-workflow", - method="POST", - json={}, - ): - current_user = _workflow_convert_api_Account(name="Current user", email="user@example.com") - current_user.id = "u1" - response = method( - api, - workflow_module.ConvertToWorkflowPayload.model_validate({}), - current_tenant_id="tenant-1", - current_user=current_user, - app_model=_workflow_convert_api_app("app-1"), - ) - - assert response["new_app_id"] == "new-app-1" - assert response["permission_keys"] == ["app.acl.view_layout", "app.acl.edit"] - - -def _make_account() -> _workflow_human_input_debug_api_Account: - account = _workflow_human_input_debug_api_Account(name="tester", email="tester@example.com") - account.status = AccountStatus.ACTIVE - account.role = TenantAccountRole.OWNER - account.id = "account-123" # type: ignore[assignment] - tenant = Tenant(name="Test tenant") - tenant.id = "tenant-123" - account._current_tenant = tenant - account._get_current_object = lambda: account # type: ignore[attr-defined] - return account - - -def _make_app(mode: AppMode) -> _workflow_human_input_debug_api_App: - return _workflow_human_input_debug_api_App( - id="app-123", - tenant_id="tenant-123", - name="Human input app", - description="", - mode=mode, - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - enable_site=True, - enable_api=True, - max_active_requests=None, - ) - - -def _patch_console_guards( + api.add_resource(Restore, "/restore") + response = app.test_client().post("/restore") + assert response.status_code == status + assert response.json == {"code": code, "status": status, "message": str(failure)} + + +def test_restore_serializes_change(workflows: Mock) -> None: + workflows.restore.return_value = WorkflowChange("hash", datetime(2024, 1, 1, tzinfo=UTC)) + assert invoke( + controller.DraftWorkflowRestoreApi, + controller.DraftWorkflowRestoreApi.post, + workflow_id="version", + ) == { + "result": "success", + "hash": "hash", + "updated_at": 1704067200, + } + + +@pytest.mark.parametrize( + ("resource", "handler", "payload", "kwargs"), + [ + ( + controller.DraftWorkflowTriggerRunApi, + controller.DraftWorkflowTriggerRunApi.post, + controller.DraftWorkflowTriggerRunPayload(node_id="node"), + {}, + ), + ( + controller.DraftWorkflowTriggerRunAllApi, + controller.DraftWorkflowTriggerRunAllApi.post, + controller.DraftWorkflowTriggerRunAllPayload(node_ids=["node"]), + {}, + ), + ( + controller.DraftWorkflowTriggerNodeApi, + controller.DraftWorkflowTriggerNodeApi.post, + None, + {"node_id": "node"}, + ), + ], +) +def test_trigger_waiting_and_error_contract( + workflows: Mock, + resource: type[Resource], + handler: Callable[..., object], + payload: BaseModel | None, + kwargs: dict[str, str], +) -> None: + args = (payload,) if payload is not None else () + workflows.trigger.return_value = None + assert invoke(resource, handler, *args, **kwargs) == {"status": "waiting", "retry_in": 2000} + workflows.trigger.side_effect = WorkflowTriggerError("plugin unavailable") + assert invoke(resource, handler, *args, **kwargs) == ({"status": "error", "error": "plugin unavailable"}, 400) + + +@pytest.mark.parametrize( + ("resource", "handler", "payload"), + [ + ( + controller.DraftWorkflowRunApi, + controller.DraftWorkflowRunApi.post, + controller.DraftWorkflowRunPayload(inputs={}), + ), + ( + controller.AdvancedChatDraftWorkflowRunApi, + controller.AdvancedChatDraftWorkflowRunApi.post, + controller.AdvancedChatWorkflowRunPayload(inputs={}), + ), + ], +) +def test_debug_run_passes_external_trace_and_stream( + app: Flask, + workflows: Mock, monkeypatch: pytest.MonkeyPatch, - account: _workflow_human_input_debug_api_Account, - app_model: _workflow_human_input_debug_api_App, + resource: type[Resource], + handler: Callable[..., object], + payload: BaseModel, ) -> None: - # Skip setup and auth guardrails - apply_config_overrides( - monkeypatch, - DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, - LOGIN_DISABLED=True, - INIT_PASSWORD="", + workflows.generate.return_value = {"task_id": "task"} + monkeypatch.setattr(controller, "get_external_trace_id", lambda request: "external-trace") + with app.test_request_context(method="POST", json={}): + response = invoke(resource, handler, payload) + assert isinstance(response, Response) + assert response.get_json() == {"task_id": "task"} + assert workflows.generate.call_args.args[2]["external_trace_id"] == "external-trace" + + +def test_convert_delegates_permissions_result(workflows: Mock) -> None: + result = {"new_app_id": "new-app", "permission_keys": ["app.acl.edit"]} + workflows.convert.return_value = result + assert ( + invoke( + controller.ConvertToWorkflowApi, + controller.ConvertToWorkflowApi.post, + controller.ConvertToWorkflowPayload(name="Copy"), + ) + == result ) - monkeypatch.setattr(login_lib, "current_user", account) - monkeypatch.setattr(login_lib, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - monkeypatch.setattr(login_lib, "check_csrf_token", lambda *_, **__: None) - monkeypatch.setattr(console_wraps, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - monkeypatch.setattr(app_wraps, "current_account_with_tenant", lambda: (account, account.current_tenant_id)) - - # Avoid hitting the database when resolving the app model - monkeypatch.setattr(app_wraps, "_load_app_model_from_scoped_session", lambda _app_id: app_model) - - -@dataclass -class PreviewCase: - resource_cls: type - path: str - mode: AppMode - - -@dataclass -class SubmitCase: - resource_cls: type - path: str - mode: AppMode - - -@dataclass -class DeliveryTestCase: - resource_cls: type - path: str - mode: AppMode - - -class TestWorkflowHumanInputDebugApiCases: - @staticmethod - @pytest.mark.parametrize( - "case", - [ - PreviewCase( - resource_cls=workflow_module.AdvancedChatDraftHumanInputFormPreviewApi, - path="/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-42/form/preview", - mode=AppMode.ADVANCED_CHAT, - ), - PreviewCase( - resource_cls=workflow_module.WorkflowDraftHumanInputFormPreviewApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-42/form/preview", - mode=AppMode.WORKFLOW, - ), - ], - ) - def test_human_input_preview_delegates_to_service( - app: Flask, monkeypatch: pytest.MonkeyPatch, case: PreviewCase - ) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - preview_payload = { - "form_id": "node-42", - "node_id": "node-42", - "node_title": "Human input", - "form_content": "
example
", - "inputs": [ - {"type": "paragraph", "output_variable_name": "topic", "default": None}, - { - "type": "select", - "output_variable_name": "choice", - "option_source": {"type": "constant", "selector": list[str](), "value": ["one", "two"]}, - }, - { - "type": "file", - "output_variable_name": "attachment", - "allowed_file_types": ["document"], - "allowed_file_extensions": list[str](), - "allowed_file_upload_methods": ["local_file", "remote_url"], - }, - { - "type": "file-list", - "output_variable_name": "attachments", - "allowed_file_types": ["image"], - "allowed_file_extensions": list[str](), - "allowed_file_upload_methods": ["local_file"], - "number_limits": 0, - }, - ], - "actions": [{"id": "continue", "title": "Continue", "button_style": "default"}], - "resolved_default_values": {"topic": ""}, - } - service_instance = MagicMock() - service_instance.get_human_input_form_preview.return_value = preview_payload - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context(case.path, method="POST", json={"inputs": {"topic": "tech"}}): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-42") - - assert response == { - **preview_payload, - "display_in_ui": None, - "form_token": None, - "expiration_time": None, - } - service_instance.get_human_input_form_preview.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-42", - inputs={"topic": "tech"}, - session=ANY, - ) - - @staticmethod - @pytest.mark.parametrize( - "case", - [ - SubmitCase( - resource_cls=workflow_module.AdvancedChatDraftHumanInputFormRunApi, - path="/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-99/form/run", - mode=AppMode.ADVANCED_CHAT, - ), - SubmitCase( - resource_cls=workflow_module.WorkflowDraftHumanInputFormRunApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-99/form/run", - mode=AppMode.WORKFLOW, - ), - ], - ) - def test_human_input_submit_forwards_payload(app: Flask, monkeypatch: pytest.MonkeyPatch, case: SubmitCase) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - result_payload = {"node_id": "node-99", "outputs": {"__rendered_content": "

done

"}, "action": "approve"} - service_instance = MagicMock() - service_instance.submit_human_input_form_preview.return_value = result_payload - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - case.path, - method="POST", - json={"form_inputs": {"answer": "42"}, "inputs": {"#node-1.result#": "LLM output"}, "action": "approve"}, - ): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-99") - - assert response == result_payload - service_instance.submit_human_input_form_preview.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-99", - form_inputs={"answer": "42"}, - inputs={"#node-1.result#": "LLM output"}, - action="approve", - session=ANY, - ) - - @staticmethod - @pytest.mark.parametrize( - "case", - [ - DeliveryTestCase( - resource_cls=workflow_module.WorkflowDraftHumanInputDeliveryTestApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-7/delivery-test", - mode=AppMode.ADVANCED_CHAT, - ), - DeliveryTestCase( - resource_cls=workflow_module.WorkflowDraftHumanInputDeliveryTestApi, - path="/console/api/apps/app-123/workflows/draft/human-input/nodes/node-7/delivery-test", - mode=AppMode.WORKFLOW, - ), - ], - ) - def test_human_input_delivery_test_calls_service( - app: Flask, monkeypatch: pytest.MonkeyPatch, case: DeliveryTestCase - ) -> None: - account = _make_account() - app_model = _make_app(case.mode) - _patch_console_guards(monkeypatch, account, app_model) - - service_instance = MagicMock() - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - case.path, - method="POST", - json={"delivery_method_id": "delivery-123"}, - ): - response = case.resource_cls().post(app_id=app_model.id, node_id="node-7") - - assert response == {} - service_instance.test_human_input_delivery.assert_called_once_with( - app_model=app_model, - account=account, - node_id="node-7", - delivery_method_id="delivery-123", - inputs={}, - session=ANY, - ) - - @staticmethod - def test_human_input_delivery_test_maps_validation_error(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - account = _make_account() - app_model = _make_app(AppMode.ADVANCED_CHAT) - _patch_console_guards(monkeypatch, account, app_model) - - service_instance = MagicMock() - service_instance.test_human_input_delivery.side_effect = ValueError("bad delivery method") - monkeypatch.setattr(workflow_module, "WorkflowService", MagicMock(return_value=service_instance)) - - with app.test_request_context( - "/console/api/apps/app-123/workflows/draft/human-input/nodes/node-1/delivery-test", - method="POST", - json={"delivery_method_id": "bad"}, - ): - with pytest.raises(ValueError, match="bad delivery method"): - workflow_module.WorkflowDraftHumanInputDeliveryTestApi().post(app_id=app_model.id, node_id="node-1") - - @staticmethod - def test_human_input_preview_rejects_non_mapping(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - account = _make_account() - app_model = _make_app(AppMode.ADVANCED_CHAT) - _patch_console_guards(monkeypatch, account, app_model) - - with app.test_request_context( - "/console/api/apps/app-123/advanced-chat/workflows/draft/human-input/nodes/node-1/form/preview", - method="POST", - json={"inputs": ["not-a-dict"]}, - ): - with pytest.raises(UnprocessableEntity): - workflow_module.AdvancedChatDraftHumanInputFormPreviewApi().post(app_id=app_model.id, node_id="node-1") + workflows.convert.assert_called_once_with(CONTEXT, str(APP_ID), {"name": "Copy"}) + + +def test_update_empty_payload_does_not_dispatch(workflows: Mock) -> None: + with pytest.raises(InvalidArgumentError, match="No valid fields to update"): + invoke( + controller.WorkflowByIdApi, + controller.WorkflowByIdApi.patch, + controller.WorkflowUpdatePayload(), + workflow_id="version", + ) + workflows.update.assert_not_called() + + +def test_delete_has_no_response_body(workflows: Mock) -> None: + assert invoke( + controller.WorkflowByIdApi, + controller.WorkflowByIdApi.delete, + workflow_id="version", + ) == (None, 204) + workflows.delete.assert_called_once_with(CONTEXT, WorkflowOwner(str(APP_ID)), "version") + + +def test_online_users_normalization_and_response(workflows: Mock) -> None: + workflows.online_users.return_value = [ + {"app_id": "a", "users": [{"user_id": "u", "username": "Name", "avatar": None}]} + ] + payload = controller.WorkflowOnlineUsersPayload(app_ids=[" a ", "", "a"]) + api = controller.WorkflowOnlineUsersApi() + result = inspect.unwrap(api.post)(api, payload, CONTEXT) + assert result == {"data": workflows.online_users.return_value} + workflows.online_users.assert_called_once_with(CONTEXT, ["a"]) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py new file mode 100644 index 00000000000000..ac956c7d84f804 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_admission.py @@ -0,0 +1,150 @@ +"""Keep the existing workflow role, RBAC and app-mode admission policy.""" + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from inspect import unwrap +from unittest.mock import create_autospec +from uuid import UUID + +import pytest +from flask import Flask +from werkzeug.exceptions import Forbidden + +from controllers.common.rbac import PlainApp, RBACCheck +from controllers.console import flask_admission +from controllers.console.app import workflow, workflow_admission +from controllers.console.app.error import AppNotFoundError, DraftWorkflowNotSync +from controllers.console.wraps import RBACPermission +from libs.login import AccountWithTenant +from machinery.context import RequestContext +from models.account import TenantAccountRole +from services.agent.workflow_contracts import WorkflowAgentBindingStore +from services.app.console_service import ConsoleAppService +from services.entities.app_entities import AppReference +from services.errors.app import WorkflowHashNotEqualError +from services.workflow.console_service import ConsoleWorkflowService +from tests.helpers.unit.config_override import apply_config_overrides +from tests.helpers.unit.model_factories import make_account + +APP_ID = UUID("11111111-1111-1111-1111-111111111111") + + +@dataclass +class Apps: + console: ConsoleAppService + + +@dataclass +class Services: + apps: Apps + console_workflows: ConsoleWorkflowService[WorkflowAgentBindingStore] + + +def admission(method: Callable[..., object]) -> Callable[..., object]: + return unwrap(method, stop=lambda view: "inject_request_context" in view.__code__.co_qualname) + + +@pytest.mark.parametrize( + ("method", "permission"), + [ + (workflow.DraftWorkflowApi.get, RBACPermission.APP_VIEW_LAYOUT), + (workflow.DraftWorkflowApi.post, RBACPermission.APP_VIEW_LAYOUT), + (workflow.PublishedWorkflowApi.post, RBACPermission.APP_RELEASE_AND_VERSION), + (workflow.DraftWorkflowRestoreApi.post, RBACPermission.APP_RELEASE_AND_VERSION), + (workflow.WorkflowByIdApi.patch, RBACPermission.APP_EDIT), + (workflow.WorkflowByIdApi.delete, RBACPermission.APP_EDIT), + (workflow.DraftWorkflowRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.DraftWorkflowTriggerRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.WorkflowDraftHumanInputFormPreviewApi.post, RBACPermission.APP_VIEW_LAYOUT), + (workflow.WorkflowDraftHumanInputFormRunApi.post, RBACPermission.APP_TEST_AND_RUN), + (workflow.DraftWorkflowNodeLastRunApi.get, RBACPermission.APP_VIEW_LAYOUT), + ], +) +def test_rbac_checks_keep_admitted_tenant_and_app( + app: Flask, monkeypatch: pytest.MonkeyPatch, method: Callable[..., object], permission: RBACPermission +) -> None: + account = make_account(role=TenantAccountRole.NORMAL) + apply_config_overrides(monkeypatch, RBAC_ENABLED=True) + monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account, "tenant")) + calls = [] + + def deny(*, tenant_id: str, account_id: str, checks: Sequence[RBACCheck], path_args: dict[str, object]) -> None: + calls.append(True) + assert (tenant_id, account_id) == ("tenant", account.id) + assert checks[0].scene == permission + assert isinstance(checks[0].locator, PlainApp) + assert path_args["app_id"] == APP_ID + raise Forbidden() + + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", deny) + with app.test_request_context(), pytest.raises(Forbidden): + admission(method)(None, app_id=APP_ID) + assert calls == [True] + + +@pytest.mark.parametrize("role", [TenantAccountRole.NORMAL, TenantAccountRole.DATASET_OPERATOR]) +@pytest.mark.parametrize( + "method", + [ + workflow.DraftWorkflowApi.get, + workflow.DraftWorkflowRunApi.post, + workflow.PublishedWorkflowApi.post, + workflow.WorkflowByIdApi.delete, + ], +) +def test_readonly_role_is_rejected_before_use_case( + app: Flask, monkeypatch: pytest.MonkeyPatch, role: TenantAccountRole, method: Callable[..., object] +) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + monkeypatch.setattr( + flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(make_account(role=role), "tenant") + ) + with app.test_request_context(), pytest.raises(Forbidden): + admission(method)(None, app_id=APP_ID) + + +@pytest.mark.parametrize("mode", ["workflow", "advanced-chat", "chat"]) +def test_context_is_stable_and_app_mode_is_checked(app: Flask, monkeypatch: pytest.MonkeyPatch, mode: str) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + account = make_account(role=TenantAccountRole.EDITOR) + monkeypatch.setattr(flask_admission, "current_account_with_tenant", lambda: AccountWithTenant(account, "tenant")) + monkeypatch.setattr(flask_admission, "get_request_id", lambda: "request") + monkeypatch.setattr(flask_admission, "get_trace_id", lambda: None) + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) + apps = create_autospec(ConsoleAppService, instance=True) + use_cases = create_autospec(ConsoleWorkflowService, instance=True) + apps.get_reference.return_value = AppReference(str(APP_ID), "App", mode, None) + use_cases.published.return_value = None + services = Services(Apps(apps), use_cases) + monkeypatch.setattr(workflow_admission, "application_services", lambda: services) + monkeypatch.setattr(workflow, "application_services", lambda: services) + with app.test_request_context(headers={"X-Trace-Id": "trace"}): + if mode == "chat": + with pytest.raises(AppNotFoundError): + admission(workflow.PublishedWorkflowApi.get)(None, app_id=APP_ID) + use_cases.published.assert_not_called() + else: + assert admission(workflow.PublishedWorkflowApi.get)(None, app_id=APP_ID) is None + context, app_id = use_cases.published.call_args.args + assert context == RequestContext("request", "trace", account.id, "tenant") + assert app_id == str(APP_ID) + + +def test_revision_conflict_is_translated_to_existing_409(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + apply_config_overrides(monkeypatch, RBAC_ENABLED=False) + monkeypatch.setattr( + flask_admission, + "current_account_with_tenant", + lambda: AccountWithTenant(make_account(role=TenantAccountRole.EDITOR), "tenant"), + ) + monkeypatch.setattr(flask_admission, "enforce_rbac_checks", lambda **_kwargs: None) + apps = create_autospec(ConsoleAppService, instance=True) + use_cases = create_autospec(ConsoleWorkflowService, instance=True) + apps.get_reference.return_value = AppReference(str(APP_ID), "App", "workflow", None) + use_cases.publish.side_effect = WorkflowHashNotEqualError() + services = Services(Apps(apps), use_cases) + monkeypatch.setattr(workflow_admission, "application_services", lambda: services) + monkeypatch.setattr(workflow, "application_services", lambda: services) + with app.test_request_context(json={}), pytest.raises(DraftWorkflowNotSync) as error: + admission(workflow.PublishedWorkflowApi.post)(None, app_id=APP_ID) + assert error.value.code == 409 diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py new file mode 100644 index 00000000000000..2c3d766aaa6f88 --- /dev/null +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_human_input_debug_api.py @@ -0,0 +1,115 @@ +"""Human input transport delegates parsed payloads to application services.""" + +from unittest.mock import Mock, create_autospec + +import pytest +from flask import Flask +from pydantic import ValidationError + +from controllers.console.app import workflow as controller +from extensions.ext_application_services import ApplicationServices +from services.human_input.debug_service import HumanInputDebugService +from tests.unit_tests.controllers.console.app.test_workflow import ( + APP_ID, + CONTEXT, + invoke, +) + + +@pytest.fixture(name="workflows") +def debug_use_cases(monkeypatch: pytest.MonkeyPatch) -> Mock: + service = create_autospec(HumanInputDebugService, instance=True) + dependencies = create_autospec(ApplicationServices, instance=True) + dependencies.human_input_debug = service + monkeypatch.setattr(controller, "application_services", lambda: dependencies) + return service + + +@pytest.mark.parametrize( + "resource", [controller.AdvancedChatDraftHumanInputFormPreviewApi, controller.WorkflowDraftHumanInputFormPreviewApi] +) +def test_preview_forwards_inputs( + app: Flask, + workflows: Mock, + resource: type[ + controller.AdvancedChatDraftHumanInputFormPreviewApi | controller.WorkflowDraftHumanInputFormPreviewApi + ], +) -> None: + payload = { + "form_id": "human", + "node_id": "human", + "node_title": "Human input", + "form_content": "Hello", + "inputs": [{"type": "paragraph", "output_variable_name": "topic", "default": None}], + "actions": [{"id": "continue", "title": "Continue", "button_style": "default"}], + "resolved_default_values": {"topic": ""}, + } + workflows.preview_form.return_value = payload + args = controller.HumanInputFormPreviewPayload(inputs={"topic": "tech"}) + with app.test_request_context(method="POST"): + assert invoke(resource, resource.post, args, node_id="human") == { + **payload, + "display_in_ui": None, + "form_token": None, + "expiration_time": None, + } + workflows.preview_form.assert_called_once_with(CONTEXT, str(APP_ID), "human", {"topic": "tech"}) + + +@pytest.mark.parametrize( + "resource", [controller.AdvancedChatDraftHumanInputFormRunApi, controller.WorkflowDraftHumanInputFormRunApi] +) +def test_submission_forwards_form_and_upstream_inputs( + app: Flask, + workflows: Mock, + resource: type[controller.AdvancedChatDraftHumanInputFormRunApi | controller.WorkflowDraftHumanInputFormRunApi], +) -> None: + outputs = {"answer": "42", "__action_id": "approve"} + workflows.submit_form.return_value = outputs + args = controller.HumanInputFormPreviewSubmitPayload( + form_inputs={"answer": "42"}, inputs={"#upstream.output#": "text"}, action="approve" + ) + with app.test_request_context(method="POST"): + assert invoke(resource, resource.post, args, node_id="human") == outputs + workflows.submit_form.assert_called_once_with( + CONTEXT, + str(APP_ID), + "human", + form_inputs={"answer": "42"}, + inputs={"#upstream.output#": "text"}, + action="approve", + ) + + +def test_delivery_forwards_default_inputs(app: Flask, workflows: Mock) -> None: + args = controller.HumanInputDeliveryTestPayload(delivery_method_id="email") + with app.test_request_context(method="POST"): + assert ( + invoke( + controller.WorkflowDraftHumanInputDeliveryTestApi, + controller.WorkflowDraftHumanInputDeliveryTestApi.post, + args, + node_id="human", + ) + == {} + ) + workflows.test_delivery.assert_called_once_with( + CONTEXT, str(APP_ID), "human", inputs={}, delivery_method_id="email" + ) + + +def test_delivery_preserves_domain_error(app: Flask, workflows: Mock) -> None: + workflows.test_delivery.side_effect = ValueError("bad delivery method") + args = controller.HumanInputDeliveryTestPayload(delivery_method_id="bad") + with app.test_request_context(method="POST"), pytest.raises(ValueError, match="bad delivery method"): + invoke( + controller.WorkflowDraftHumanInputDeliveryTestApi, + controller.WorkflowDraftHumanInputDeliveryTestApi.post, + args, + node_id="human", + ) + + +def test_preview_rejects_non_mapping_inputs() -> None: + with pytest.raises(ValidationError): + controller.HumanInputFormPreviewPayload.model_validate({"inputs": ["not-a-dict"]}) diff --git a/api/tests/unit_tests/controllers/console/app/test_workflow_run.py b/api/tests/unit_tests/controllers/console/app/test_workflow_run.py index d53bf8a124960e..2e672a5413999e 100644 --- a/api/tests/unit_tests/controllers/console/app/test_workflow_run.py +++ b/api/tests/unit_tests/controllers/console/app/test_workflow_run.py @@ -35,6 +35,7 @@ DifyAPISQLAlchemyWorkflowNodeExecutionRepository, ) from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository +from repositories.tools.provider_repository import ToolProviderRepository from services.enterprise.rbac_service import RBACService from services.workflow_node_execution_trace_service import WorkflowNodeExecutionTrace from services.workflow_run_service import WorkflowRunPauseDetails, WorkflowRunPausedNode, WorkflowRunService @@ -461,13 +462,13 @@ def test_workflow_run_detail_maps_missing_run_to_not_found( @staticmethod def test_workflow_run_node_executions_return_frontend_trace_contract( - app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers ) -> None: _account(sqlite_session) execution = _workflow_run_node_execution(sqlite_session) workflow_runs = Mock() workflow_runs.get_workflow_run_node_executions.return_value = [ - node_execution_response_source(execution, session=sqlite_session) + node_execution_response_source(execution, session=sqlite_session, tool_providers=tool_providers) ] _workflow_run_api_mock_application_services(monkeypatch, workflow_runs) monkeypatch.setattr(db, "session", lambda: sqlite_session) @@ -517,11 +518,14 @@ def test_workflow_run_node_executions_return_frontend_trace_contract( @staticmethod @pytest.fixture - def workflow_run_service(sqlite_session_factory: sessionmaker[Session]) -> WorkflowRunService: + def workflow_run_service( + sqlite_session_factory: sessionmaker[Session], tool_providers: ToolProviderRepository + ) -> WorkflowRunService: return WorkflowRunService( session_factory=sqlite_session_factory, workflow_runs=DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory), node_executions=DifyAPISQLAlchemyWorkflowNodeExecutionRepository(sqlite_session_factory), + tool_providers=tool_providers, ) @staticmethod diff --git a/api/tests/unit_tests/controllers/console/app/test_wraps.py b/api/tests/unit_tests/controllers/console/app/test_wraps.py index 6336e071620209..a271fa65f59ed9 100644 --- a/api/tests/unit_tests/controllers/console/app/test_wraps.py +++ b/api/tests/unit_tests/controllers/console/app/test_wraps.py @@ -1,6 +1,8 @@ from __future__ import annotations +from collections.abc import Callable from inspect import getsource +from typing import cast from uuid import uuid4 import pytest @@ -8,7 +10,6 @@ from controllers.common.session import with_session from controllers.console.app import completion as completion_module -from controllers.console.app import workflow as workflow_module from controllers.console.app import wraps as wraps_module from controllers.console.app.error import AppNotFoundError from models.agent import Agent, AgentScope, AgentSource, AgentStatus @@ -35,10 +36,10 @@ def test_get_app_model_injects_model(monkeypatch: pytest.MonkeyPatch, sqlite_ses monkeypatch.setattr(wraps_module.db, "session", sqlite_session) @wraps_module.get_app_model - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id - assert handler(app_id=app_model.id) == app_model.id + assert cast(Callable[..., str], handler)(app_id=app_model.id) == app_model.id def test_get_app_model_rejects_wrong_mode(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session) -> None: @@ -47,20 +48,20 @@ def test_get_app_model_rejects_wrong_mode(monkeypatch: pytest.MonkeyPatch, sqlit monkeypatch.setattr(wraps_module.db, "session", sqlite_session) @wraps_module.get_app_model(mode=[AppMode.COMPLETION]) - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id with pytest.raises(AppNotFoundError): - handler(app_id=app_model.id) + cast(Callable[..., str], handler)(app_id=app_model.id) def test_get_app_model_requires_app_id() -> None: @wraps_module.get_app_model - def handler(app_model): + def handler(app_model: App) -> str: return app_model.id with pytest.raises(ValueError, match="missing app_id in path parameters"): - handler() + cast(Callable[..., str], handler)() def test_wraps_with_session_reexports_common_session_decorator() -> None: @@ -76,14 +77,15 @@ def test_get_app_model_prefers_injected_session( class Handler: @wraps_module.get_app_model - def get(self, _injected_session, app_model): + def get(self, _injected_session: Session, app_model: App) -> str: return app_model.id # An unbound real Session fails on query, so success proves the injected # request Session was preferred over the legacy scoped-session fallback. with Session() as scoped_session: monkeypatch.setattr(wraps_module.db, "session", scoped_session) - assert Handler().get(sqlite_session, app_id=app_model.id) == app_model.id + get = cast(Callable[..., str], Handler().get) + assert get(sqlite_session, app_id=app_model.id) == app_model.id @pytest.mark.parametrize("scope", [AgentScope.ROSTER, AgentScope.WORKFLOW_ONLY]) @@ -119,11 +121,12 @@ def get(self, *_args: object, app_model: App) -> str: return app_model.id args = (sqlite_session,) if inject_session else () + get = cast(Callable[..., str], Handler().get) if workflow_only: with pytest.raises(AppNotFoundError): - Handler().get(*args, app_id=app_model.id) + get(*args, app_id=app_model.id) else: - assert Handler().get(*args, app_id=app_model.id) == app_model.id + assert get(*args, app_id=app_model.id) == app_model.id @pytest.mark.parametrize( @@ -131,11 +134,7 @@ def get(self, *_args: object, app_model: App) -> str: [ completion_module.CompletionMessageApi, completion_module.ChatMessageApi, - workflow_module.AdvancedChatDraftWorkflowRunApi, - workflow_module.DraftWorkflowRunApi, - workflow_module.DraftWorkflowTriggerRunApi, - workflow_module.DraftWorkflowTriggerRunAllApi, ], ) -def test_migrated_handlers_open_session_before_app_lookup(resource: type) -> None: +def test_legacy_handlers_open_session_before_app_lookup(resource: type) -> None: assert "@with_session\n @get_app_model" in getsource(resource) diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py index 74f3ee45dce732..6927e3b8935ce7 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_datasource_content_preview.py @@ -1,10 +1,11 @@ from inspect import unwrap -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, patch import pytest from flask import Flask from controllers.console import console_ns +from controllers.console.datasets.rag_pipeline import datasource_content_preview as preview_module from controllers.console.datasets.rag_pipeline.datasource_content_preview import ( DataSourceContentPreviewApi, DatasourceContentPreviewPayload, @@ -39,21 +40,19 @@ def test_post_success(self, app: Flask): preview_result = {"content": "preview data"} - service_instance = MagicMock() - service_instance.run_datasource_node_preview.return_value = preview_result - with ( app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload), - patch( - "controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService", - return_value=service_instance, - ), + patch.object( + preview_module.application_services().knowledge.pipeline_execution, + "run_datasource_node_preview", + return_value=preview_result, + ) as preview, ): req_data = DatasourceContentPreviewPayload.model_validate(payload) response, status = method(api, req_data, account, pipeline, node_id) - service_instance.run_datasource_node_preview.assert_called_once_with( + preview.assert_called_once_with( pipeline=pipeline, node_id=node_id, user_inputs=payload["inputs"], @@ -98,20 +97,18 @@ def test_post_without_credential_id(self, app: Flask): pipeline = Pipeline(tenant_id="tenant-id", name="Test Pipeline") account = make_account() - service_instance = MagicMock() - service_instance.run_datasource_node_preview.return_value = {"ok": True} - with ( app.test_request_context("/", json=payload), patch.object(type(console_ns), "payload", payload), - patch( - "controllers.console.datasets.rag_pipeline.datasource_content_preview.RagPipelineService", - return_value=service_instance, - ), + patch.object( + preview_module.application_services().knowledge.pipeline_execution, + "run_datasource_node_preview", + return_value={"ok": True}, + ) as preview, ): req_data = DatasourceContentPreviewPayload.model_validate(payload) response, status = method(api, req_data, account, pipeline, "node-1") - service_instance.run_datasource_node_preview.assert_called_once() + preview.assert_called_once() assert status == 200 assert response == {"ok": True} diff --git a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py index a7b601cd2e6322..ed78a389bf1c3b 100644 --- a/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py +++ b/api/tests/unit_tests/controllers/console/datasets/rag_pipeline/test_rag_pipeline_workflow.py @@ -1,14 +1,13 @@ from __future__ import annotations import json -from collections.abc import Generator, Iterator -from contextlib import contextmanager +from collections.abc import Iterator from dataclasses import dataclass from datetime import datetime from inspect import unwrap from inspect import unwrap as unwrap_all from typing import TypedDict, Unpack -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, create_autospec, patch from uuid import UUID, uuid4 import pytest @@ -16,11 +15,11 @@ from pydantic import ValidationError from sqlalchemy import Engine from sqlalchemy.orm import Session, scoped_session, sessionmaker -from werkzeug.exceptions import BadRequest, Forbidden, HTTPException, NotFound +from werkzeug.exceptions import HTTPException import models.workflow as workflow_models import services -import services.rag_pipeline.rag_pipeline as rag_pipeline_service +from controllers.common.errors import AccessDeniedError, InvalidRequestError, NotFoundError from controllers.console.app.error import DraftWorkflowNotExist, DraftWorkflowNotSync from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as module from controllers.console.datasets.rag_pipeline import rag_pipeline_workflow as workflow_controller @@ -55,25 +54,32 @@ WorkflowUpdatePayload, ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.datasource.entities.datasource_entities import DatasourceParameter from core.plugin.entities import OAuthSchema from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity from core.plugin.impl.datasource import PluginDatasourceManager +from extensions.ext_application_services import ApplicationServices from fields.workflow_run_fields import node_execution_response_source from graphon.enums import WorkflowNodeExecutionStatus from libs.datetime_utils import naive_utc_now +from machinery.context import RequestContext from models.account import Account, TenantAccountRole from models.dataset import Dataset, Pipeline from models.engine import db from models.enums import CreatorUserRole, PermissionEnum from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.workflow.definition_repository import workflow_record, workflow_snapshot from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError from services.errors.llm import InvokeRateLimitError from services.errors.rag_pipeline import RagPipelineResourceNotFoundError +from services.errors.workflow_service import WorkflowInUseError from services.rag_pipeline.rag_pipeline import RagPipelineService from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService +from services.workflow.contracts import WorkflowOwner, WorkflowSnapshot +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from tests.helpers.unit.config_override import config_overrides_context from tests.helpers.unit.controllers.error_body import raised_error from tests.helpers.unit.model_factories import make_account as _rag_pipeline_workflow_make_account @@ -536,7 +542,7 @@ def test_rag_pipeline_transform_rejects_read_only_member(sqlite_engine: Engine) with ( config_overrides_context(RBAC_ENABLED=False), - pytest.raises(Forbidden), + pytest.raises(AccessDeniedError), ): handler( api, session, _RAG_PIPELINE_WORKFLOW_DEFAULT_WORKFLOW_TENANT_ID, account, UUID(DEFAULT_DATASET_ID) @@ -554,7 +560,7 @@ def test_rag_pipeline_transform_rejects_dataset_from_another_tenant_before_servi with ( patch.object(module.RagPipelineTransformService, "transform_dataset") as transform_dataset, - pytest.raises(NotFound), + pytest.raises(NotFoundError), ): handler( api, @@ -579,7 +585,7 @@ def test_rag_pipeline_transform_enforces_legacy_dataset_permission_before_servic with ( config_overrides_context(RBAC_ENABLED=False), patch.object(module.RagPipelineTransformService, "transform_dataset") as transform_dataset, - pytest.raises(Forbidden), + pytest.raises(AccessDeniedError), ): handler( api, @@ -633,7 +639,7 @@ def test_rag_pipeline_transform_maps_missing_pipeline_to_not_found(sqlite_engine "transform_dataset", side_effect=RagPipelineResourceNotFoundError("Pipeline not found"), ), - pytest.raises(NotFound, match="Pipeline not found"), + pytest.raises(NotFoundError, match="Pipeline not found"), ): handler( api, @@ -687,7 +693,7 @@ def test_rag_pipeline_transform_skips_legacy_acl_when_rbac_is_enabled(sqlite_eng ), ], ) - def test_rag_pipeline_run_uses_sqlite_session( + def test_rag_pipeline_run_uses_injected_generator( app: Flask, sqlite_engine: Engine, pipeline_application: PipelineGenerator, @@ -714,7 +720,8 @@ def test_rag_pipeline_run_uses_sqlite_session( assert response == {"ok": True} load_pipeline.assert_called_once_with(session, pipeline.id) - assert generate.call_args.kwargs["session"] is session + assert generate.call_args.kwargs["pipeline"] is pipeline + assert generate.call_args.kwargs["streaming"] is (api_type is module.DraftRagPipelineRunApi) assert generate.call_args.kwargs["generator"] is pipeline_application assert session.get_bind() is sqlite_engine @@ -752,7 +759,7 @@ def test_rag_pipeline_run_translates_rate_limit( ): handler(api, req_data, session, _account(), pipeline.id) - assert generate.call_args.kwargs["session"] is session + assert generate.call_args.kwargs["pipeline"] is pipeline assert generate.call_args.kwargs["generator"] is pipeline_application @@ -963,7 +970,6 @@ def sqlite_database( database = SQLiteDatabase(engine=sqlite_engine, session=database_session) monkeypatch.setattr(workflow_controller, "db", database) monkeypatch.setattr(workflow_models, "db", database) - monkeypatch.setattr(rag_pipeline_service, "db", database) with database_session() as session: default_author = Account(name="Default Author", email="default-author@example.com") @@ -994,7 +1000,7 @@ def test_get_draft_success(self, app: Flask, workflow_author: Account, sqlite_se workflow = make_workflow(created_by=workflow_author.id) service = MagicMock() - service.get_draft_workflow.return_value = workflow + service.get_draft_workflow_record.return_value = workflow_record(workflow, sqlite_session) with ( app.test_request_context("/"), @@ -1022,7 +1028,7 @@ def test_get_draft_not_exist(self, app: Flask, sqlite_session: Session) -> None: pipeline = make_pipeline() service = MagicMock() - service.get_draft_workflow.return_value = None + service.get_draft_workflow_record.return_value = None with ( app.test_request_context("/"), @@ -1034,25 +1040,24 @@ def test_get_draft_not_exist(self, app: Flask, sqlite_session: Session) -> None: with pytest.raises(DraftWorkflowNotExist): method(api, sqlite_session, pipeline) - def test_sync_hash_not_match(self, app: Flask) -> None: + def test_sync_hash_not_match(self, app: Flask, workflow_application: ApplicationServices) -> None: api = DraftRagPipelineApi() method = unwrap(api.post) pipeline = make_pipeline() user = _rag_pipeline_workflow_apis_make_account() - service = MagicMock() - service.sync_draft_workflow.side_effect = WorkflowHashNotEqualError() + service = create_autospec(WorkflowDraftService, instance=True) + service.sync.side_effect = WorkflowHashNotEqualError() with ( app.test_request_context("/", json={"graph": empty_mapping(), "features": empty_mapping()}), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): with pytest.raises(DraftWorkflowNotSync): - method(api, user, pipeline) + method(api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id) def test_sync_invalid_text_plain(self, app: Flask) -> None: api = DraftRagPipelineApi() @@ -1064,10 +1069,14 @@ def test_sync_invalid_text_plain(self, app: Flask) -> None: with ( app.test_request_context("/", data="bad-json", headers={"Content-Type": "text/plain"}), ): - response, status = raised_error(lambda: method(api, user, pipeline)) + response, status = raised_error( + lambda: method(api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id) + ) assert status == 400 - def test_restore_published_workflow_to_draft_success(self, app: Flask) -> None: + def test_restore_published_workflow_to_draft_success( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) @@ -1078,62 +1087,70 @@ def test_restore_published_workflow_to_draft_success(self, app: Flask) -> None: created_at=datetime(2024, 1, 1), ) - service = MagicMock() - service.restore_published_workflow_to_draft.return_value = workflow + service = create_autospec(WorkflowDraftService, instance=True) + service.restore.return_value = workflow_snapshot(workflow) with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): - result = method(api, user, pipeline, "published-workflow") + result = method( + api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id, "published-workflow" + ) assert result["result"] == "success" assert result["hash"] == workflow.unique_hash - def test_restore_published_workflow_to_draft_not_found(self, app: Flask) -> None: + def test_restore_published_workflow_to_draft_not_found( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) pipeline = make_pipeline() user = _rag_pipeline_workflow_apis_make_account(id="account-1") - service = MagicMock() - service.restore_published_workflow_to_draft.side_effect = WorkflowNotFoundError("Workflow not found") + service = create_autospec(WorkflowDraftService, instance=True) + service.restore.side_effect = WorkflowNotFoundError("Workflow not found") with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): - with pytest.raises(NotFound): - method(api, user, pipeline, "published-workflow") - - def test_restore_published_workflow_to_draft_returns_400_for_draft_source(self, app: Flask) -> None: + with pytest.raises(NotFoundError): + method( + api, + RequestContext("test", None, user.id, pipeline.tenant_id), + pipeline.id, + "published-workflow", + ) + + def test_restore_published_workflow_to_draft_returns_400_for_draft_source( + self, app: Flask, workflow_application: ApplicationServices + ) -> None: api = RagPipelineDraftWorkflowRestoreApi() method = unwrap(api.post) pipeline = make_pipeline() user = _rag_pipeline_workflow_apis_make_account(id="account-1") - service = MagicMock() - service.restore_published_workflow_to_draft.side_effect = IsDraftWorkflowError( - "source workflow must be published" - ) + service = create_autospec(WorkflowDraftService, instance=True) + service.restore.side_effect = IsDraftWorkflowError("source workflow must be published") with ( app.test_request_context("/", method="POST"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.workflow_drafts, "restore", service.restore), + patch.object(workflow_application.workflow_drafts, "sync", service.sync), ): with pytest.raises(HTTPException) as exc: - method(api, user, pipeline, "draft-workflow") + method( + api, RequestContext("test", None, user.id, pipeline.tenant_id), pipeline.id, "draft-workflow" + ) assert exc.value.code == 400 assert exc.value.description == "source workflow must be published" @@ -1174,7 +1191,7 @@ def test_iteration_node_conversation_not_exists(self, app: Flask) -> None: side_effect=services.errors.conversation.ConversationNotExistsError(), ), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): method(api, NodeRunPayload(), user, pipeline, "node") def test_loop_node_success(self, app: Flask) -> None: @@ -1205,14 +1222,12 @@ def test_execution_not_found(self, app: Flask) -> None: pipeline = make_pipeline() user = _rag_pipeline_workflow_apis_make_account() - service = MagicMock() - service.run_draft_workflow_node.return_value = None - with ( app.test_request_context("/", json={"inputs": empty_mapping()}), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, + patch.object( + workflow_controller.application_services().knowledge.pipeline_execution, + "run_draft_workflow_node", + return_value=None, ), ): with pytest.raises(ValueError, match="Workflow node execution not found"): @@ -1254,35 +1269,42 @@ def test_get_without_published_workflow_returns_null(self, app: Flask, is_publis assert result is None - def test_publish_success(self, app: Flask) -> None: + @pytest.mark.parametrize("outcome", ["success", "conflict", "missing", "invalid"]) + def test_publish( + self, + app: Flask, + workflow_application: ApplicationServices, + monkeypatch: pytest.MonkeyPatch, + outcome: str, + ) -> None: api = PublishedRagPipelineApi() method = unwrap(api.post) - - tenant_id = str(uuid4()) - pipeline = Pipeline( - tenant_id=tenant_id, - name="test-pipeline", - description="test", - created_by=str(uuid4()), - ) - user = _rag_pipeline_workflow_apis_make_account(id="u1") - - workflow = make_workflow(id=str(uuid4()), created_at=naive_utc_now()) - - service = MagicMock() - service.publish_workflow.return_value = workflow - - with ( - app.test_request_context("/"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, - ), - ): - result = method(api, user, pipeline) - - assert result["result"] == "success" - assert "created_at" in result + context = RequestContext("test", None, "account-1", "tenant-1") + calls: list[tuple[RequestContext, str]] = [] + workflow = workflow_snapshot(make_workflow(id="published", created_at=naive_utc_now())) + + def publish(context: RequestContext, pipeline_id: str) -> WorkflowSnapshot: + calls.append((context, pipeline_id)) + if outcome == "invalid": + raise workflow_controller.RagPipelinePublicationError("Invalid configuration") + if outcome == "conflict": + raise workflow_controller.WorkflowHashNotEqualError() + if outcome == "missing": + raise workflow_controller.WorkflowNotFoundError("Pipeline not found") + return workflow + + monkeypatch.setattr(workflow_controller, "application_services", lambda: workflow_application) + monkeypatch.setattr(workflow_application.knowledge.pipeline_publication, "publish", publish) + with app.test_request_context("/", method="POST"): + if outcome == "success": + result = method(api, context, "pipeline-1") + assert result["result"] == "success" + assert "created_at" in result + else: + with pytest.raises(HTTPException) as error: + method(api, context, "pipeline-1") + assert error.value.code == {"conflict": 409, "missing": 404, "invalid": 400}[outcome] + assert calls == [(context, "pipeline-1")] class TestMiscApis: def test_task_stop(self, app: Flask) -> None: @@ -1344,7 +1366,12 @@ def test_recommended_plugins(self, app: Flask) -> None: ): result = method(api, RagPipelineRecommendedPluginQuery(type="all"), tenant_id, user) assert result == recommended_plugins - service.get_recommended_plugins.assert_called_once_with("all", user, tenant_id) + service.get_recommended_plugins.assert_called_once_with( + "all", + user, + tenant_id, + tool_providers=workflow_controller.application_services().tools.tool_providers, + ) class TestDefaultBlockConfigApi: def test_get_block_config_success(self, app: Flask) -> None: @@ -1377,7 +1404,7 @@ def test_get_block_config_invalid_json(self, app: Flask) -> None: method(api, DefaultBlockConfigQuery(q="bad-json"), pipeline, "llm") class TestPublishedAllRagPipelineApi: - def test_get_published_workflows_success(self, app: Flask) -> None: + def test_get_published_workflows_success(self, app: Flask, sqlite_session: Session) -> None: api = PublishedAllRagPipelineApi() method = unwrap(api.get) @@ -1385,7 +1412,10 @@ def test_get_published_workflows_success(self, app: Flask) -> None: user = _rag_pipeline_workflow_apis_make_account(id="u1") service = MagicMock() - service.get_all_published_workflow.return_value = ([make_workflow(id="w1")], False) + service.get_all_published_workflow.return_value = ( + [workflow_record(make_workflow(id="w1"), sqlite_session)], + False, + ) with ( app.test_request_context("/"), @@ -1410,11 +1440,11 @@ def test_get_published_workflows_forbidden(self, app: Flask) -> None: with ( app.test_request_context("/?user_id=u2"), ): - with pytest.raises(Forbidden): + with pytest.raises(AccessDeniedError): method(api, WorkflowListQuery(user_id="u2"), user, pipeline) class TestRagPipelineByIdApi: - def test_patch_success(self, app: Flask) -> None: + def test_patch_success(self, app: Flask, sqlite_session: Session) -> None: api = RagPipelineByIdApi() method = unwrap(api.patch) @@ -1424,7 +1454,7 @@ def test_patch_success(self, app: Flask) -> None: workflow = make_workflow(id="w1", marked_name="test") service = MagicMock() - service.update_workflow.return_value = workflow + service.update_workflow.return_value = workflow_record(workflow, sqlite_session) payload = {"marked_name": "test"} @@ -1453,82 +1483,25 @@ def test_patch_no_fields(self, app: Flask) -> None: assert status == 400 assert result == {"code": "invalid_param", "message": "No valid fields to update"} - @pytest.mark.parametrize("transaction_fails", [False, True], ids=["commit-succeeds", "commit-fails"]) - def test_delete_retires_candidates_only_after_transaction_exit( - self, - app: Flask, - transaction_fails: bool, + @pytest.mark.parametrize("in_use", [False, True]) + def test_delete_delegates_to_application( + self, app: Flask, workflow_application: ApplicationServices, in_use: bool ) -> None: api = RagPipelineByIdApi() method = unwrap(api.delete) - - pipeline = make_pipeline(tenant_id="t1", workflow_id="active-workflow") - user = _rag_pipeline_workflow_apis_make_account() - - events: list[str] = [] - error = RuntimeError("commit failed") - workflow_service = MagicMock() - workflow_service.delete_workflow.side_effect = lambda **_kwargs: events.append("delete") or ["inline-agent"] - transaction_factory = MagicMock() - - @contextmanager - def transaction() -> Generator[Session]: - events.append("transaction-enter") - with Session() as session: - yield session - events.append("transaction-exit") - if transaction_fails: - raise error - - transaction_factory.begin.side_effect = transaction - + context = RequestContext("delete", None, "account", "tenant") + delete = Mock(side_effect=WorkflowInUseError("currently in use by pipeline") if in_use else None) with ( app.test_request_context("/", method="DELETE"), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.WorkflowService", - return_value=workflow_service, - ), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.sessionmaker", - return_value=transaction_factory, - ), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow." - "WorkflowAgentRetirementService.retire_unowned" - ) as retire_unowned, + patch.object(workflow_controller, "application_services", return_value=workflow_application), + patch.object(workflow_application.console_workflows, "delete", delete), ): - retire_unowned.side_effect = lambda **_kwargs: events.append("retire") - if transaction_fails: - with pytest.raises(RuntimeError) as exc_info: - method(api, user, pipeline, "old-workflow") - assert exc_info.value is error + if in_use: + with pytest.raises(InvalidRequestError, match="currently in use by pipeline"): + method(api, context, "pipeline", "version") else: - result = method(api, user, pipeline, "old-workflow") - assert result == (None, 204) - - workflow_service.delete_workflow.assert_called_once() - assert events == ["transaction-enter", "delete", "transaction-exit"] + ( - [] if transaction_fails else ["retire"] - ) - if transaction_fails: - retire_unowned.assert_not_called() - else: - retire_unowned.assert_called_once_with( - tenant_id=pipeline.tenant_id, - agent_ids=["inline-agent"], - account_id=user.id, - ) - - def test_delete_active_workflow_rejected(self, app: Flask) -> None: - api = RagPipelineByIdApi() - method = unwrap(api.delete) - - pipeline = make_pipeline(tenant_id="t1", workflow_id="active-workflow") - user = _rag_pipeline_workflow_apis_make_account() - - with app.test_request_context("/", method="DELETE"): - with pytest.raises(BadRequest, match="currently in use by pipeline"): - method(api, user, pipeline, "active-workflow") + assert method(api, context, "pipeline", "version") == (None, 204) + delete.assert_called_once_with(context, WorkflowOwner("pipeline", "pipeline"), "version") class TestRagPipelineWorkflowLastRunApi: def test_last_run_success(self, app: Flask) -> None: @@ -1571,11 +1544,17 @@ def test_last_run_not_found(self, app: Flask) -> None: return_value=service, ), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): method(api, pipeline, "node1") class TestRagPipelineWorkflowRunNodeExecutionListApi: - def test_get_node_executions_passes_current_user(self, app: Flask, sqlite_session: Session) -> None: + def test_get_node_executions_passes_current_user( + self, + app: Flask, + sqlite_session: Session, + *, + tool_providers: ToolProviderRepository, + ) -> None: api = RagPipelineWorkflowRunNodeExecutionListApi() method = unwrap(api.get) @@ -1594,7 +1573,11 @@ def test_get_node_executions_passes_current_user(self, app: Flask, sqlite_sessio service = MagicMock() service.get_rag_pipeline_workflow_run_node_executions.return_value = [ - node_execution_response_source(node_exec, session=sqlite_session) + node_execution_response_source( + node_exec, + session=sqlite_session, + tool_providers=tool_providers, + ) ] with ( @@ -1610,6 +1593,7 @@ def test_get_node_executions_passes_current_user(self, app: Flask, sqlite_sessio pipeline=pipeline, run_id=str(run_id), user=user, + tool_providers=workflow_controller.application_services().tools.tool_providers, ) assert result["data"][0]["id"] == "node-exec-1" assert result["data"][0]["inputs"] == {"query": "hello"} @@ -1632,14 +1616,12 @@ def test_set_datasource_variables_success(self, app: Flask) -> None: "start_node_title": "Node", } - service = MagicMock() - service.set_datasource_variables.return_value = make_node_execution(node_id="n1") - with ( app.test_request_context("/", json=payload), - patch( - "controllers.console.datasets.rag_pipeline.rag_pipeline_workflow.RagPipelineService", - return_value=service, + patch.object( + workflow_controller.application_services().knowledge.pipeline_execution, + "set_datasource_variables", + return_value=make_node_execution(node_id="n1"), ), ): result = method(api, DatasourceVariablesPayload.model_validate(payload), user, pipeline) diff --git a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py index ff2e0e1d100ab9..acef2626ed3c15 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_datasets_document.py @@ -1,17 +1,21 @@ import datetime as _datasets_document_datetime import json +from collections.abc import Iterator from datetime import datetime as _datasets_document_download_datetime from inspect import unwrap from io import BytesIO +from os import PathLike from pathlib import Path from types import SimpleNamespace +from typing import Never from unittest.mock import MagicMock, create_autospec, patch from uuid import UUID from zipfile import ZipFile import pytest -from flask import send_file -from sqlalchemy import event +from flask import Flask, Response, send_file +from sqlalchemy import Engine, event +from sqlalchemy.orm import Session, sessionmaker from werkzeug.exceptions import Forbidden, NotFound from controllers.common.errors import InvalidArgumentError, NotFoundError @@ -54,9 +58,12 @@ InvalidActionError, InvalidMetadataError, ) +from extensions.storage.storage_type import StorageType from machinery.context import RequestContext from models.dataset import Dataset, Document +from models.enums import CreatorUserRole from models.model import UploadFile +from services.errors.document import DocumentSourceNotFoundError from services.knowledge.dataset_access import ( AccessibleDataset, DatasetAccess, @@ -610,7 +617,7 @@ def test_document_mutation_routes_require_operation_specific_permissions(method, FILE_ID = str(UUID(int=2)) -def _document(**values): +def _document(**values: object) -> Document: return Document( **{ "id": DOC_ID, @@ -631,19 +638,21 @@ def _document(**values): class TestDatasetsDocumentDownloadCases: @staticmethod @pytest.fixture(autouse=True) - def downloads(monkeypatch, sqlite_session_factory): + def downloads( + monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session] + ) -> DatasetDocumentApplicationService: with sqlite_session_factory.begin() as session: session.add(Dataset(id="dataset-1", tenant_id="tenant-1", name="Dataset", created_by="actor-1")) session.add(_document()) upload = UploadFile( tenant_id="tenant-1", - storage_type="local", + storage_type=StorageType.LOCAL, key="key-1", name="document.txt", size=5, extension="txt", mime_type="text/plain", - created_by_role="account", + created_by_role=CreatorUserRole.ACCOUNT, created_by="actor-1", created_at=_datasets_document_download_datetime(2024, 1, 1), used=False, @@ -663,12 +672,12 @@ def downloads(monkeypatch, sqlite_session_factory): return documents @staticmethod - def test_document_download_returns_signed_url(app, monkeypatch): + def test_document_download_returns_signed_url(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: from services.knowledge import dataset_service - signed = [] + signed: list[dict[str, object]] = [] - def sign(**kwargs): + def sign(**kwargs: object) -> str: signed.append(kwargs) return "https://files.example/download" @@ -682,56 +691,66 @@ def sign(**kwargs): @staticmethod @pytest.mark.parametrize( - "values", + ("values", "expected_error"), [ - {"data_source_type": "website_crawl"}, - {"data_source_info": "{}"}, - {"data_source_info": json.dumps({"upload_file_id": "missing"})}, - {"tenant_id": "foreign"}, - {"dataset_id": "foreign"}, + ({"data_source_type": "website_crawl"}, DocumentSourceNotFoundError), + ({"data_source_info": "{}"}, DocumentSourceNotFoundError), + ({"data_source_info": json.dumps({"upload_file_id": "missing"})}, DocumentSourceNotFoundError), + ({"tenant_id": "foreign"}, NotFound), + ({"dataset_id": "foreign"}, NotFound), ], ) - def test_invalid_or_foreign_document_is_not_downloadable(app, sqlite_session_factory, values): + def test_invalid_or_foreign_document_is_not_downloadable( + app: Flask, + sqlite_session_factory: sessionmaker[Session], + values: dict[str, object], + expected_error: type[BaseException], + ) -> None: with sqlite_session_factory.begin() as session: document = session.get(Document, DOC_ID) + assert document is not None for key, value in values.items(): setattr(document, key, value) - with app.test_request_context("/"), pytest.raises(NotFound): + with app.test_request_context("/"), pytest.raises(expected_error): unwrap(controller.DocumentDownloadApi.get)( controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID ) @staticmethod - def test_cross_tenant_upload_file_is_not_signed(app, sqlite_session_factory): + def test_cross_tenant_upload_file_is_not_signed(app: Flask, sqlite_session_factory: sessionmaker[Session]) -> None: with sqlite_session_factory.begin() as session: - session.get(UploadFile, FILE_ID).tenant_id = "foreign" - with app.test_request_context("/"), pytest.raises(NotFound): + upload = session.get(UploadFile, FILE_ID) + assert upload is not None + upload.tenant_id = "foreign" + with app.test_request_context("/"), pytest.raises(DocumentSourceNotFoundError): unwrap(controller.DocumentDownloadApi.get)( controller.DocumentDownloadApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1", DOC_ID ) @staticmethod - def test_zip_is_readable_and_cleanup_waits_for_response_close(app, monkeypatch, sqlite_engine): + def test_zip_is_readable_and_cleanup_waits_for_response_close( + app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine + ) -> None: from services import file_service - paths = [] - checked_out = set() + paths: list[Path] = [] + checked_out: set[object] = set() - def checkout(_connection, record, _proxy): + def checkout(_connection: object, record: object, _proxy: object) -> None: checked_out.add(record) - def checkin(_connection, record): + def checkin(_connection: object, record: object) -> None: checked_out.discard(record) - def load(key, *, stream): + def load(key: str, *, stream: bool) -> Iterator[bytes]: assert not checked_out, "storage I/O must run after the read transaction closes" assert key == "key-1" assert stream is True return iter([b"hello"]) - def send(path, **kwargs): + def send(path: str | PathLike[str], *, mimetype: str, as_attachment: bool, download_name: str) -> Response: paths.append(Path(path)) - return send_file(path, **kwargs) + return send_file(path, mimetype=mimetype, as_attachment=as_attachment, download_name=download_name) monkeypatch.setattr(file_service.storage, "load", load) monkeypatch.setattr(controller, "send_file", send) @@ -753,13 +772,13 @@ def send(path, **kwargs): event.remove(sqlite_engine, "checkin", checkin) @staticmethod - def test_zip_cleanup_when_response_creation_fails(app, monkeypatch): + def test_zip_cleanup_when_response_creation_fails(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: from services import file_service - paths = [] + paths: list[Path] = [] monkeypatch.setattr(file_service.storage, "load", lambda *_args, **_kwargs: iter([b"hello"])) - def fail_send(path, **_kwargs): + def fail_send(path: str | PathLike[str], **_kwargs: object) -> Never: paths.append(Path(path)) raise RuntimeError("response failed") @@ -775,10 +794,12 @@ def fail_send(path, **_kwargs): assert not paths[0].exists() @staticmethod - def test_zip_rejects_non_upload_document(app, sqlite_session_factory): + def test_zip_rejects_non_upload_document(app: Flask, sqlite_session_factory: sessionmaker[Session]) -> None: with sqlite_session_factory.begin() as session: - session.get(Document, DOC_ID).data_source_type = "website_crawl" - with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(NotFound): + document = session.get(Document, DOC_ID) + assert document is not None + document.data_source_type = "website_crawl" + with app.test_request_context("/", json={"document_ids": [DOC_ID]}), pytest.raises(DocumentSourceNotFoundError): unwrap(controller.DocumentBatchDownloadZipApi.post)( controller.DocumentBatchDownloadZipApi(), _DATASETS_DOCUMENT_DOWNLOAD_CONTEXT, "dataset-1" ) diff --git a/api/tests/unit_tests/controllers/console/explore/test_completion.py b/api/tests/unit_tests/controllers/console/explore/test_completion.py index 57683b593545cf..064eb703849434 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_completion.py +++ b/api/tests/unit_tests/controllers/console/explore/test_completion.py @@ -15,12 +15,14 @@ from models.enums import ConversationFromSource from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.errors.app_model_config import AppModelConfigBrokenError from services.errors.conversation import ConversationCompletedError, ConversationNotExistsError from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, _Harness, @@ -424,6 +426,9 @@ def test_resource_removed_after_admission_is_revalidated_before_generation( sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, deleted_resource: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, ) -> None: def delete_resource() -> None: with sqlite_session_factory.begin() as session: @@ -439,7 +444,11 @@ def delete_resource() -> None: installed_apps=_InstalledAppCompletionInstalledappservices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, + variables=workflow_variables, + runtime=workflow_runtime, + ), ) ) ) @@ -806,6 +815,9 @@ def test_chat_conversation_preflight_returns_404_before_starting_generation_and_ sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rejection: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, ) -> None: conversation_id = str(uuid4()) with sqlite_session_factory.begin() as session: @@ -830,7 +842,11 @@ def test_chat_conversation_preflight_returns_404_before_starting_generation_and_ installed_apps=_CompletionInstalledAppServices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, + variables=workflow_variables, + runtime=workflow_runtime, + ), ) ) ) diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial.py b/api/tests/unit_tests/controllers/console/explore/test_trial.py index 56937256b30e76..cd7a55f136b601 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial.py @@ -44,7 +44,6 @@ from controllers.console.app import preview_admission as _trial_preview_details_admission_module from controllers.console.explore import trial from core.agent.entities import AgentToolEntity -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.app.entities.app_invoke_entities import InvokeFrom from core.errors.error import ( AppInvokeQuotaExceededError, @@ -58,7 +57,6 @@ from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ToolParameter from core.tools.errors import ToolSSRFError -from core.tools.tool_manager import ToolManager from core.workflow.llm_environment_variable import LLMEnvironmentVariable from enums import DeploymentEdition from extensions.ext_login import DifyLoginManager, unauthorized_handler @@ -104,10 +102,13 @@ from repositories.app_definition_query_repository import AppDefinitionQueryRepository from repositories.app_preview_query_repository import AppPreviewQueryRepository from repositories.recommended_app_catalog_repository import DatabaseRecommendedAppCatalogRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from repositories.trial_app_repository import TrialAppRepository from services.account_errors import AccountNotFoundError from services.agent.errors import AgentVersionNotFoundError from services.agent_config_service import AgentConfigService +from services.app.generation.errors import AgentAppNotPublishedError from services.app_definition_query_service import AppDefinitionQueryService, AppDefinitionUnavailableError from services.app_preview_details_adapters import AppPreviewDetailsRuntime from services.app_preview_query_service import AppPreviewQueryService @@ -133,6 +134,7 @@ RecommendedAppQueryService, ) from services.remote_file_service import RemoteFileService +from services.tools.tool_manager import ToolManager from services.trial_app_access_service import TrialAppAccessService, TrialAppRef from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationService @@ -703,6 +705,18 @@ def csrf(_request: Request, account_id: str) -> None: api.add_resource(trial_module.TrialChatTextApi, "/trial-apps//text-to-audio") return _TrialAudioHarness(app, account, target, trial, sqlite_session_factory, runtime, state) + @staticmethod + def test_multipart_body_limit_is_handled_before_service_error_translation( + harness: _TrialAudioHarness, + ) -> None: + harness.app.config["MAX_CONTENT_LENGTH"] = 8 + + response = harness.post(_ASR) + + _trial_audio_assert_error(response, 413, "request_entity_too_large") + assert harness.runtime.calls == [] + assert harness.usage() is None + @staticmethod def test_asr_upload_keeps_transcript_and_owner_tenant(harness: _TrialAudioHarness) -> None: response = harness.post(_ASR) @@ -2687,7 +2701,17 @@ def decrypt(self, tenant_id: str, token: str) -> str: assert tenant_id == self.viewer_workspace_id return "tool-secret" - def tool_runtime(self, *, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + self, + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, + ) -> Tool: + del workflow_queries, tool_providers assert isinstance(self.engine.pool, QueuePool) assert self.engine.pool.checkedout() == 0 assert tenant_id == self.viewer_workspace_id @@ -2786,7 +2810,14 @@ def legacy_detail(self) -> dict[str, object]: app = session.get(_trial_preview_details_App, self.target.id) assert app is not None return trial_module.TrialAppDetailResponse.model_validate( - AppResponseView(app, session=session, account=self.viewer), from_attributes=True + AppResponseView( + app, + session=session, + account=self.viewer, + tool_providers=ToolProviderRepository(self.factory), + workflow_queries=WorkflowToolRepository(self.factory), + ), + from_attributes=True, ).model_dump(mode="json") @@ -2806,6 +2837,9 @@ def harness( config_overrides: Callable[..., None], sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> _TrialPreviewDetailsHarness: config_overrides( LOGIN_DISABLED=False, @@ -2935,7 +2969,11 @@ def track_session(session: Session, _transaction: SessionTransaction, _connectio app_previews=AppPreviewQueryService( apps=AppPreviewQueryRepository(session_factory=factory), is_previewable=recommendations.is_previewable ), - app_preview_details=AppPreviewDetailsRuntime(details=AppPreviewQueryRepository(session_factory=factory)), + app_preview_details=AppPreviewDetailsRuntime( + details=AppPreviewQueryRepository(session_factory=factory), + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ), recommended_app_queries=recommendations, ) for module in (trial_module, _trial_preview_details_admission_module): @@ -3105,10 +3143,25 @@ def hardcoded_provider(provider_id: str) -> object: return object() raise KeyError(provider_id) - def tool_runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, + ) -> Tool: if agent_tool.provider_id != "example": raise ValueError("Invalid historical provider ID") - return harness.io.tool_runtime(tenant_id=tenant_id, app_id=app_id, agent_tool=agent_tool, user_id=user_id) + return harness.io.tool_runtime( + tenant_id=tenant_id, + app_id=app_id, + agent_tool=agent_tool, + user_id=user_id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ) plugin_queries: list[str] = [] @@ -3184,10 +3237,23 @@ def get_runtime_parameters(self) -> list[ToolParameter]: SimpleNamespace(parameters=[parameter], identity=SimpleNamespace(name="broken")) ) - def tool_runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def tool_runtime( + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, + ) -> Tool: if agent_tool.provider_id == "example": return harness.io.tool_runtime( - tenant_id=tenant_id, app_id=app_id, agent_tool=agent_tool, user_id=user_id + tenant_id=tenant_id, + app_id=app_id, + agent_tool=agent_tool, + user_id=user_id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, ) if failure == "runtime": raise RuntimeError("Tool provider is unavailable") diff --git a/api/tests/unit_tests/controllers/console/explore/test_workflow.py b/api/tests/unit_tests/controllers/console/explore/test_workflow.py index f9e710204a598d..43dd82e027aa26 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_workflow.py +++ b/api/tests/unit_tests/controllers/console/explore/test_workflow.py @@ -19,12 +19,14 @@ from models.workflow import WorkflowType as ModelWorkflowType from repositories.installed_app_repository import SQLAlchemyInstalledAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskControlService from services.errors.llm import InvokeRateLimitError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse, InstalledAppGenerationService +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.unit_tests.controllers.console.explore.test_installed_app_admission import ( _assert_json_response, _Harness, @@ -412,6 +414,9 @@ def test_workflow_run_revalidates_app_while_stop_uses_admission_snapshot( sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, action: str, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, ) -> None: def remove_app() -> None: with sqlite_session_factory.begin() as session: @@ -424,7 +429,9 @@ def remove_app() -> None: installed_apps=_InstalledAppServices( generation=InstalledAppGenerationService( usage=SQLAlchemyInstalledAppRepository(session_factory=sqlite_session_factory), - runtime=AppGenerateServiceRuntime(session_factory=sqlite_session_factory), + runtime=AppGenerateServiceRuntime( + session_factory=sqlite_session_factory, variables=workflow_variables, runtime=workflow_runtime + ), ) ), app_tasks=AppTaskControlService(redis_client=stop_redis), diff --git a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py index eef3224942c8f1..1c00ff7d6ea896 100644 --- a/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py +++ b/api/tests/unit_tests/controllers/console/snippets/test_snippet_workflow.py @@ -2,17 +2,23 @@ import json from inspect import unwrap +from typing import NoReturn, TypedDict, Unpack from unittest.mock import Mock import pytest from flask import Flask from sqlalchemy.orm import Session -from werkzeug.exceptions import HTTPException, NotFound +from werkzeug.exceptions import HTTPException +from controllers.common.errors import NotFoundError from controllers.console.snippets import snippet_workflow as snippet_workflow_module +from extensions.ext_application_services import ApplicationServices +from machinery.context import RequestContext from models.account import Account, TenantAccountRole from models.snippet import CustomizedSnippet from models.workflow import Workflow +from repositories.workflow.definition_repository import workflow_record, workflow_snapshot +from services.workflow.contracts import WorkflowRecord from tests.helpers.unit.controllers.error_body import raised_error from tests.helpers.unit.model_factories import make_account @@ -26,20 +32,28 @@ def _account(account_id: str = "account-1") -> Account: ) -def _snippet(**overrides) -> CustomizedSnippet: - data = { - "id": "snippet-1", - "tenant_id": "tenant-1", - "name": "Snippet", - "description": "Description", - "type": "node", - "created_by": "account-1", - } - data.update(overrides) - return CustomizedSnippet(**data) +class _SnippetOverrides(TypedDict, total=False): + is_published: bool + input_fields: str | None -def _workflow(**overrides) -> Workflow: +def _snippet(**overrides: Unpack[_SnippetOverrides]) -> CustomizedSnippet: + snippet = CustomizedSnippet( + id="snippet-1", + tenant_id="tenant-1", + name="Snippet", + description="Description", + type="node", + created_by="account-1", + ) + if "is_published" in overrides: + snippet.is_published = overrides["is_published"] + if "input_fields" in overrides: + snippet.input_fields = overrides["input_fields"] + return snippet + + +def _workflow(**overrides: object) -> Workflow: workflow = Workflow.new( tenant_id="tenant-1", app_id="snippet-1", @@ -58,9 +72,9 @@ def _workflow(**overrides) -> Workflow: return workflow -def test_get_snippet_requires_snippet_id(app): +def test_get_snippet_requires_snippet_id(app: Flask) -> None: @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs with app.test_request_context("/snippets"): @@ -72,7 +86,7 @@ def test_get_snippet_injects_resolved_snippet(app: Flask, monkeypatch: pytest.Mo snippet = _snippet() @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs["snippet"] monkeypatch.setattr( @@ -90,7 +104,7 @@ def view(**kwargs): def test_get_snippet_raises_not_found_when_snippet_missing(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: @snippet_workflow_module.get_snippet - def view(**kwargs): + def view(**kwargs: object) -> object: return kwargs monkeypatch.setattr( @@ -101,7 +115,7 @@ def view(**kwargs): monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_snippet_by_id", Mock(return_value=None)) with app.test_request_context("/snippets/snippet-1"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): view(snippet_id="snippet-1") @@ -109,7 +123,7 @@ def test_draft_workflow_get_raises_when_missing( app: Flask, monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: snippet = _snippet() - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow", Mock(return_value=None)) + monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow_record", Mock(return_value=None)) api = snippet_workflow_module.SnippetDraftWorkflowApi() handler = unwrap(api.get) @@ -119,18 +133,17 @@ def test_draft_workflow_get_raises_when_missing( handler(api, unbound_session, snippet=snippet) -def test_draft_workflow_get_uses_session_aware_response_source( +def test_draft_workflow_get_serializes_materialized_record( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: workflow = _workflow(updated_by="account-2") sqlite_session.add_all([_account(), _account("account-2")]) sqlite_session.commit() snippet = _snippet() - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_draft_workflow", Mock(return_value=workflow)) monkeypatch.setattr( - snippet_workflow_module.WorkflowAgentPublishService, - "project_draft_bindings_to_graph", - Mock(return_value=workflow.graph_dict), + snippet_workflow_module.SnippetService, + "get_draft_workflow_record", + Mock(return_value=workflow_record(workflow, sqlite_session)), ) api = snippet_workflow_module.SnippetDraftWorkflowApi() @@ -145,11 +158,13 @@ def test_draft_workflow_get_uses_session_aware_response_source( assert response["tool_published"] is False -def test_draft_workflow_post_returns_400_for_invalid_graph(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_draft_workflow_post_returns_400_for_invalid_graph( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: user = _account("account-1") snippet = _snippet() sync_draft_workflow = Mock(side_effect=ValueError("invalid graph")) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "sync_draft_workflow", sync_draft_workflow) + monkeypatch.setattr(workflow_application.workflow_drafts, "sync", sync_draft_workflow) api = snippet_workflow_module.SnippetDraftWorkflowApi() handler = unwrap(api.post) @@ -162,11 +177,8 @@ def test_draft_workflow_post_returns_400_for_invalid_graph(app: Flask, monkeypat response, status_code = raised_error( lambda: handler( api, - snippet_workflow_module.SnippetDraftSyncPayload.model_validate( - {"graph": {"nodes": [], "edges": []}, "hash": "hash-1"} - ), - user, - snippet, + RequestContext("test", None, user.id, snippet.tenant_id), + snippet.id, ) ) @@ -174,7 +186,7 @@ def test_draft_workflow_post_returns_400_for_invalid_graph(app: Flask, monkeypat assert response == {"code": "invalid_param", "message": "invalid graph"} -def test_draft_config_returns_parallel_depth_limit(app) -> None: +def test_draft_config_returns_parallel_depth_limit(app: Flask) -> None: api = snippet_workflow_module.SnippetDraftConfigApi() handler = unwrap(api.get) @@ -182,7 +194,7 @@ def test_draft_config_returns_parallel_depth_limit(app) -> None: assert handler(api, snippet=_snippet()) == {"parallel_depth_limit": 3} -def test_published_workflow_get_returns_none_when_not_published(app, unbound_session: Session) -> None: +def test_published_workflow_get_returns_none_when_not_published(app: Flask, unbound_session: Session) -> None: api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.get) @@ -190,14 +202,18 @@ def test_published_workflow_get_returns_none_when_not_published(app, unbound_ses assert handler(api, unbound_session, snippet=_snippet(is_published=False)) is None -def test_published_workflow_get_uses_session_aware_response_source( +def test_published_workflow_get_serializes_materialized_record( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: workflow = _workflow(updated_by="account-2") sqlite_session.add_all([_account(), _account("account-2")]) sqlite_session.commit() snippet = _snippet(is_published=True) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_published_workflow", Mock(return_value=workflow)) + monkeypatch.setattr( + snippet_workflow_module.SnippetService, + "get_published_workflow_record", + Mock(return_value=workflow_record(workflow, sqlite_session)), + ) api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.get) @@ -221,9 +237,7 @@ def test_published_workflow_post_returns_400_when_publish_fails( sqlite_session.add(snippet) sqlite_session.commit() - def fail_publish(*, session: Session, snippet: CustomizedSnippet, account: Account): - snippet.name = "Uncommitted name" - session.add(snippet) + def fail_publish(*, snippet: CustomizedSnippet, account: Account) -> NoReturn: raise ValueError("No valid workflow found.") monkeypatch.setattr(snippet_workflow_module.SnippetService, "publish_workflow", Mock(side_effect=fail_publish)) @@ -232,7 +246,7 @@ def fail_publish(*, session: Session, snippet: CustomizedSnippet, account: Accou handler = unwrap(api.post) with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): - response, status_code = raised_error(lambda: handler(api, sqlite_session, user, snippet)) + response, status_code = raised_error(lambda: handler(api, user, snippet)) assert status_code == 400 assert response == {"code": "invalid_param", "message": "No valid workflow found."} @@ -255,11 +269,25 @@ def test_published_workflow_post_returns_success( api = snippet_workflow_module.SnippetPublishedWorkflowApi() handler = unwrap(api.post) with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): - response = handler(api, sqlite_session, user, snippet) + response = handler(api, user, snippet) assert response["result"] == "success" +def test_published_workflow_post_translates_preparation_conflict(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + from controllers.console.app.error import DraftWorkflowNotSync + from services.errors.app import WorkflowHashNotEqualError + + monkeypatch.setattr( + snippet_workflow_module.SnippetService, "publish_workflow", Mock(side_effect=WorkflowHashNotEqualError) + ) + api = snippet_workflow_module.SnippetPublishedWorkflowApi() + with app.test_request_context("/snippets/snippet-1/workflows/publish", method="POST", json={}): + with pytest.raises(DraftWorkflowNotSync) as error: + unwrap(api.post)(api, _account("account-1"), _snippet()) + assert error.value.code == 409 + + def test_default_block_configs_delegates_to_service(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: get_default_block_configs = Mock(return_value=[{"type": "llm"}]) monkeypatch.setattr(snippet_workflow_module.SnippetService, "get_default_block_configs", get_default_block_configs) @@ -284,7 +312,9 @@ def test_list_published_snippet_workflows_includes_input_fields( snippet = _snippet(input_fields=json.dumps(input_fields)) monkeypatch.setattr( - snippet_workflow_module.SnippetService, "get_all_published_workflows", Mock(return_value=([workflow], False)) + snippet_workflow_module.SnippetService, + "get_all_published_workflows", + Mock(return_value=([workflow_record(workflow, sqlite_session)], False)), ) api = snippet_workflow_module.SnippetPublishedAllWorkflowApi() @@ -301,14 +331,14 @@ def test_list_published_snippet_workflows_includes_input_fields( assert response["items"][0]["input_fields"] == input_fields -def test_restore_published_snippet_workflow_to_draft_success(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_restore_published_snippet_workflow_to_draft_success( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: workflow = _workflow(updated_at=None) user = _account("account-1") snippet = _snippet() - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "restore_published_workflow_to_draft", Mock(return_value=workflow) - ) + monkeypatch.setattr(workflow_application.workflow_drafts, "restore", Mock(return_value=workflow_snapshot(workflow))) api = snippet_workflow_module.SnippetDraftWorkflowRestoreApi() handler = unwrap(api.post) @@ -317,19 +347,23 @@ def test_restore_published_snippet_workflow_to_draft_success(app: Flask, monkeyp "/snippets/snippet-1/workflows/published-workflow/restore", method="POST", ): - response = handler(api, user, snippet, workflow_id="published-workflow") + response = handler( + api, RequestContext("test", None, user.id, snippet.tenant_id), snippet.id, workflow_id="published-workflow" + ) assert response["result"] == "success" assert response["hash"] == workflow.unique_hash -def test_restore_published_snippet_workflow_to_draft_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: +def test_restore_published_snippet_workflow_to_draft_not_found( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices +) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=snippet_workflow_module.WorkflowNotFoundError("Workflow not found")), ) @@ -340,19 +374,24 @@ def test_restore_published_snippet_workflow_to_draft_not_found(app: Flask, monke "/snippets/snippet-1/workflows/published-workflow/restore", method="POST", ): - with pytest.raises(NotFound): - handler(api, user, snippet, workflow_id="published-workflow") + with pytest.raises(NotFoundError): + handler( + api, + RequestContext("test", None, user.id, snippet.tenant_id), + snippet.id, + workflow_id="published-workflow", + ) def test_restore_published_snippet_workflow_to_draft_returns_400_for_draft_source( - app, monkeypatch: pytest.MonkeyPatch + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices ) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=snippet_workflow_module.IsDraftWorkflowError("source workflow must be published")), ) @@ -364,21 +403,23 @@ def test_restore_published_snippet_workflow_to_draft_returns_400_for_draft_sourc method="POST", ): with pytest.raises(HTTPException) as exc: - handler(api, user, snippet, workflow_id="draft-workflow") + handler( + api, RequestContext("test", None, user.id, snippet.tenant_id), snippet.id, workflow_id="draft-workflow" + ) assert exc.value.code == 400 assert exc.value.description == snippet_workflow_module.RESTORE_SOURCE_WORKFLOW_MUST_BE_PUBLISHED_MESSAGE def test_restore_published_snippet_workflow_to_draft_returns_400_for_invalid_graph( - app, monkeypatch: pytest.MonkeyPatch + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices ) -> None: user = _account("account-1") snippet = _snippet() monkeypatch.setattr( - snippet_workflow_module.SnippetService, - "restore_published_workflow_to_draft", + workflow_application.workflow_drafts, + "restore", Mock(side_effect=ValueError("invalid snippet workflow graph")), ) @@ -390,7 +431,12 @@ def test_restore_published_snippet_workflow_to_draft_returns_400_for_invalid_gra method="POST", ): with pytest.raises(HTTPException) as exc: - handler(api, user, snippet, workflow_id="published-workflow") + handler( + api, + RequestContext("test", None, user.id, snippet.tenant_id), + snippet.id, + workflow_id="published-workflow", + ) assert exc.value.code == 400 assert exc.value.description == "invalid snippet workflow graph" @@ -408,10 +454,10 @@ def test_update_published_snippet_workflow_returns_updated_workflow( sqlite_session.add(snippet) sqlite_session.commit() - def update_persisted_snippet(*, session: Session, snippet: CustomizedSnippet, **_kwargs): + def update_persisted_snippet(*, session: Session, snippet: CustomizedSnippet, **_kwargs: object) -> WorkflowRecord: merged_snippet = session.merge(snippet) merged_snippet.description = "Updated in transaction" - return workflow + return workflow_record(workflow, session) update_workflow = Mock(side_effect=update_persisted_snippet) monkeypatch.setattr(snippet_workflow_module.SnippetService, "update_workflow", update_workflow) @@ -479,7 +525,7 @@ def test_update_published_snippet_workflow_raises_not_found( sqlite_session.add(snippet) sqlite_session.commit() - def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_kwargs): + def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_kwargs: object) -> None: merged_snippet = session.merge(snippet) merged_snippet.name = "Rolled back name" @@ -495,7 +541,7 @@ def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_ method="PATCH", json={"marked_name": "v1"}, ): - with pytest.raises(NotFound, match="Workflow not found"): + with pytest.raises(NotFoundError, match="Workflow not found"): handler( api, snippet_workflow_module.WorkflowUpdatePayload.model_validate({"marked_name": "v1"}), @@ -510,60 +556,33 @@ def update_missing_workflow(*, session: Session, snippet: CustomizedSnippet, **_ assert snippet.name == "Snippet" -def test_delete_published_snippet_workflow_succeeds(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - snippet = _snippet() - delete_workflow = Mock(return_value=True) - monkeypatch.setattr(snippet_workflow_module.SnippetService, "delete_workflow", delete_workflow) - - api = snippet_workflow_module.SnippetWorkflowByIdApi() - handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/workflow-1", method="DELETE"): - response, status_code = handler(api, snippet, workflow_id="workflow-1") - - assert status_code == 204 - assert response is None - delete_workflow.assert_called_once() - delete_call = delete_workflow.call_args.kwargs - assert isinstance(delete_call["session"], Session) - assert delete_call["snippet"] is snippet - assert delete_call["workflow_id"] == "workflow-1" - - -def test_delete_published_snippet_workflow_raises_not_found(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: - def delete_missing_workflow(**_kwargs): - raise ValueError("Workflow with ID missing-workflow not found") - - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "delete_workflow", Mock(side_effect=delete_missing_workflow) - ) - - api = snippet_workflow_module.SnippetWorkflowByIdApi() - handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/missing-workflow", method="DELETE"): - with pytest.raises(NotFound): - handler(api, _snippet(), workflow_id="missing-workflow") - - -def test_delete_published_snippet_workflow_raises_bad_request_when_in_use( - app: Flask, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("outcome", ["success", "missing", "active", "draft"]) +def test_delete_published_snippet_workflow( + app: Flask, monkeypatch: pytest.MonkeyPatch, workflow_application: ApplicationServices, outcome: str ) -> None: - def delete_active_workflow(**_kwargs): - raise snippet_workflow_module.WorkflowInUseError("Cannot delete workflow that is currently in use") - - monkeypatch.setattr( - snippet_workflow_module.SnippetService, "delete_workflow", Mock(side_effect=delete_active_workflow) - ) - + calls: list[tuple[RequestContext, object, str]] = [] + context = RequestContext("test", None, "account-1", "tenant-1") + + def delete(context: RequestContext, owner: snippet_workflow_module.WorkflowOwner, workflow_id: str) -> None: + calls.append((context, owner, workflow_id)) + if outcome == "missing": + raise snippet_workflow_module.WorkflowNotFoundError("Workflow not found") + if outcome == "active": + raise snippet_workflow_module.WorkflowInUseError("Workflow is in use") + if outcome == "draft": + raise snippet_workflow_module.DraftWorkflowDeletionError("Cannot delete draft") + + monkeypatch.setattr(workflow_application.console_workflows, "delete", delete) api = snippet_workflow_module.SnippetWorkflowByIdApi() handler = unwrap(api.delete) - - with app.test_request_context("/snippets/snippet-1/workflows/workflow-1", method="DELETE"): - with pytest.raises(HTTPException) as exc_info: - handler(api, _snippet(), workflow_id="workflow-1") - - assert exc_info.value.code == 400 + with app.test_request_context("/", method="DELETE"): + if outcome == "success": + assert handler(api, context, "snippet-1", "workflow-1") == (None, 204) + else: + with pytest.raises(HTTPException) as error: + handler(api, context, "snippet-1", "workflow-1") + assert error.value.code == (404 if outcome == "missing" else 400) + assert calls == [(context, snippet_workflow_module.WorkflowOwner("snippet-1", "snippet"), "workflow-1")] def test_workflow_run_detail_raises_not_found_when_run_missing( @@ -576,7 +595,7 @@ def test_workflow_run_detail_raises_not_found_when_run_missing( handler = unwrap(api.get) with app.test_request_context("/snippets/snippet-1/workflow-runs/run-1"): - with pytest.raises(NotFound, match="Workflow run not found"): + with pytest.raises(NotFoundError, match="Workflow run not found"): handler(api, sqlite_session, snippet=snippet, run_id="run-1") @@ -592,7 +611,7 @@ def test_draft_node_last_run_raises_not_found_when_execution_missing( handler = unwrap(api.get) with app.test_request_context("/snippets/snippet-1/workflows/draft/nodes/llm-1/last-run"): - with pytest.raises(NotFound, match="Node last run not found"): + with pytest.raises(NotFoundError, match="Node last run not found"): handler(api, snippet=snippet, node_id="llm-1") diff --git a/api/tests/unit_tests/controllers/console/workspace/test_snippets.py b/api/tests/unit_tests/controllers/console/workspace/test_snippets.py index ea9a6a57859f3f..5b9f56b4e2216d 100644 --- a/api/tests/unit_tests/controllers/console/workspace/test_snippets.py +++ b/api/tests/unit_tests/controllers/console/workspace/test_snippets.py @@ -7,8 +7,8 @@ from flask import Flask from pydantic import ValidationError from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.console.workspace import snippets as snippets_module from models.account import Account, TenantAccountRole from models.snippet import CustomizedSnippet @@ -174,10 +174,8 @@ def test_create_snippet_defaults_unknown_type_and_returns_created( assert create_snippet.call_args.kwargs["snippet_type"] == snippets_module.SnippetType.NODE -def test_create_snippet_rejects_forbidden_nodes(app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_create_snippet_rejects_forbidden_nodes(app: Flask, sqlite_session: Session): user = _account("account-1") - create_snippet = Mock() - monkeypatch.setattr(snippets_module.SnippetService, "create_snippet", create_snippet) req_data = snippets_module.CreateSnippetPayload( name="snippet with invalid node", @@ -211,7 +209,7 @@ def test_create_snippet_rejects_forbidden_nodes(app: Flask, monkeypatch: pytest. assert status_code == 400 assert "knowledge-retrieval" in response["message"] - create_snippet.assert_not_called() + assert sqlite_session.query(CustomizedSnippet).count() == 0 def test_get_snippet_detail_raises_when_missing(app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): @@ -221,7 +219,7 @@ def test_get_snippet_detail_raises_when_missing(app: Flask, monkeypatch: pytest. handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -387,7 +385,7 @@ def patch_through_decorator(resource, session: Session, snippet_id: str): method="PATCH", json={"name": "New"}, ): - with pytest.raises(BadRequest, match="name already in use"): + with pytest.raises(InvalidRequestError, match="name already in use"): patch_through_decorator(api, snippet_id="snippet-1") assert _persisted_name(sqlite_session) == "Snippet" @@ -457,7 +455,7 @@ def test_export_snippet_returns_yaml_attachment(app: Flask, monkeypatch: pytest. monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(export_snippet_dsl=export_snippet_dsl)), ) @@ -485,7 +483,7 @@ def test_export_snippet_raises_not_found_for_missing_workflow( monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock( return_value=SimpleNamespace( export_snippet_dsl=Mock(side_effect=ValueError("Missing published workflow workflow-1")) @@ -496,7 +494,7 @@ def test_export_snippet_raises_not_found_for_missing_workflow( handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1/export?workflow_id=workflow-1"): - with pytest.raises(NotFound, match="Missing published workflow workflow-1"): + with pytest.raises(NotFoundError, match="Missing published workflow workflow-1"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -509,7 +507,7 @@ def test_import_snippet_returns_202_for_pending_confirmation( import_snippet = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(import_snippet=import_snippet)), ) @@ -539,7 +537,7 @@ def test_import_snippet_returns_400_for_failed_import( import_snippet = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(import_snippet=import_snippet)), ) @@ -568,7 +566,7 @@ def test_import_confirm_returns_200_for_completed_import( confirm_import = Mock(return_value=result) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(confirm_import=confirm_import)), ) @@ -595,7 +593,7 @@ def test_check_dependencies_raises_when_snippet_missing( handler = unwrap(api.get) with app.test_request_context("/workspaces/current/customized-snippets/snippet-1/check-dependencies"): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") @@ -609,7 +607,7 @@ def test_check_dependencies_returns_dependency_result( monkeypatch.setattr(snippets_module.SnippetService, "get_snippet_by_id", Mock(return_value=snippet)) monkeypatch.setattr( snippets_module, - "SnippetDslService", + "build_snippet_dsl_service", Mock(return_value=SimpleNamespace(check_dependencies=check_dependencies)), ) @@ -636,7 +634,7 @@ def test_increment_use_count_raises_when_snippet_missing( "/workspaces/current/customized-snippets/snippet-1/use-count/increment", method="POST", ): - with pytest.raises(NotFound, match="Snippet not found"): + with pytest.raises(NotFoundError, match="Snippet not found"): handler(api, sqlite_session, "tenant-1", snippet_id="snippet-1") diff --git a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py index ce082dcfc0b601..bfd4fd753083be 100644 --- a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py +++ b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py @@ -2,28 +2,34 @@ from collections.abc import Generator from contextlib import contextmanager -from unittest.mock import MagicMock, patch +from dataclasses import dataclass +from unittest.mock import MagicMock, Mock, create_autospec, patch import pytest from flask import Flask from redis.exceptions import LockNotOwnedError, RedisError from redis.lock import Lock -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from controllers.inner_api import bp as inner_api_bp from core.plugin.entities.plugin_daemon import CredentialType from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin from core.tools.plugin_tool.provider import PluginToolProviderController -from core.tools.tool_manager import ToolManager from extensions.application_services.tools import resolve_tool_credentials +from extensions.application_services.workflow import build_workflow_execution_dependencies from extensions.ext_redis import RedisClientWrapper from models.model import App, AppMode from models.tools import BuiltinToolProvider +from repositories.app.agent_app_repository import AgentAppRepository +from services.agent.tool_invocation_service import AgentToolInnerService from services.credentials.refresh import CredentialRefreshUnavailableError from services.credentials.tool_runtime import ToolCredentialRefreshError from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime -from services.entities.agent_tool_inner import AgentToolInvokeResponse +from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse from services.errors.agent_tool_inner import AgentToolInnerServiceError +from services.tools.agent_invocation_gateway import AgentToolInvocationGateway +from services.tools.tool_manager import ToolManager +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.config_override import config_overrides_context @@ -35,6 +41,7 @@ def _headers(api_key: str | None = "inner-key") -> dict[str, str]: def _payload(*, provider_type: str = "plugin", credential_id: str | None = "credential-1") -> dict[str, object]: + """Accept a provider selection and return a valid trusted-caller request payload without external effects.""" return { "caller": { "tenant_id": "tenant-1", @@ -69,6 +76,24 @@ def _agent_inner_auth() -> Generator[None]: yield +@dataclass(frozen=True) +class _AgentApps: + tools: AgentToolInnerService + + +@dataclass(frozen=True) +class _ApplicationServices: + agent_apps: _AgentApps + + +@contextmanager +def _tool_service() -> Generator[Mock]: + service = create_autospec(AgentToolInnerService, instance=True, spec_set=True) + services = _ApplicationServices(agent_apps=_AgentApps(tools=service)) + with patch("controllers.inner_api.agent.tools.application_services", return_value=services): + yield service.invoke + + def test_post_returns_service_response() -> None: app = Flask(__name__) app.config["TESTING"] = True @@ -76,7 +101,7 @@ def test_post_returns_service_response() -> None: with ( _agent_inner_auth(), - patch("controllers.inner_api.agent.tools.AgentToolInnerService.invoke") as mock_invoke, + _tool_service() as mock_invoke, ): mock_invoke.return_value = AgentToolInvokeResponse( messages=[{"type": "text", "message": {"text": "ok"}}], @@ -94,6 +119,7 @@ def test_post_returns_service_response() -> None: body = response.get_json() assert body["observation"] == "ok" assert body["metadata"]["provider_type"] == "plugin" + mock_invoke.assert_called_once_with(request=AgentToolInvokeRequest.model_validate(_payload())) def test_post_returns_400_for_invalid_body() -> None: @@ -120,7 +146,7 @@ def test_post_preserves_service_error_status_code_and_description() -> None: with ( _agent_inner_auth(), - patch("controllers.inner_api.agent.tools.AgentToolInnerService.invoke") as mock_invoke, + _tool_service() as mock_invoke, ): mock_invoke.side_effect = AgentToolInnerServiceError( error_code="app_tenant_mismatch", @@ -139,15 +165,28 @@ def test_post_preserves_service_error_status_code_and_description() -> None: assert body["message"] == "App does not belong to the caller tenant." +@pytest.mark.parametrize("api_key", [None, "incorrect-key"]) +def test_post_rejects_untrusted_callers_before_service_invocation(api_key: str | None) -> None: + app = Flask(__name__) + app.config["TESTING"] = True + app.register_blueprint(inner_api_bp) + with _agent_inner_auth(), _tool_service() as invoke: + response = app.test_client().post("/inner/api/agent/tools/invoke", json=_payload(), headers=_headers(api_key)) + assert response.status_code == 404 + invoke.assert_not_called() + + @pytest.mark.parametrize("credential_id", [None, "1a1db43c-e9d8-4ef2-af8f-38b25424a4ad"]) @pytest.mark.parametrize("failure", ["busy", "redis", "lost", "rejected"]) -def test_post_preserves_refresh_error_classification_through_real_credential_resolution( +def test_post_preserves_refresh_error_classification_through_runtime_ports( sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch, credential_id: str | None, failure: str, ) -> None: + """Given an admitted Agent caller, preserve credential errors and stored credentials through the runtime ports.""" provider_id = "langgenius/search/search" credential = BuiltinToolProvider( tenant_id="tenant-1", @@ -209,6 +248,15 @@ def test_post_preserves_refresh_error_classification_through_real_credential_res plugin_unique_identifier="langgenius/search:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", tenant_id="tenant-1", ) + variables = create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True) + service = AgentToolInnerService( + apps=AgentAppRepository(session_factory=sqlite_session_factory), + tools=AgentToolInvocationGateway( + variables=variables, + runtime=build_workflow_execution_dependencies(sqlite_session_factory), + ), + ) + services = _ApplicationServices(agent_apps=_AgentApps(tools=service)) redis, _ = redis_transport monkeypatch.setattr("extensions.application_services.tools.redis_client", redis) monkeypatch.setattr("core.helper.provider_cache.redis_client", redis) @@ -220,8 +268,9 @@ def test_post_preserves_refresh_error_classification_through_real_credential_res with ( _agent_inner_auth(), + patch("controllers.inner_api.agent.tools.application_services", return_value=services), patch.object(ToolManager, "get_builtin_provider", return_value=controller), - patch("core.helper.credential_utils.check_credential_policy_compliance"), + patch("services.credentials.tool_runtime_adapters.runtime_check_credential_policy_compliance"), patch.object( Lock, "acquire", diff --git a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py index 337a023563d04f..31ff1181078526 100644 --- a/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py +++ b/api/tests/unit_tests/controllers/inner_api/plugin/test_plugin.py @@ -8,11 +8,12 @@ """ import inspect -from unittest.mock import MagicMock, patch +import json +from unittest.mock import MagicMock, Mock, create_autospec, patch import pytest from flask import Flask -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from controllers.inner_api.plugin import plugin as plugin_module from controllers.inner_api.plugin.plugin import ( @@ -36,11 +37,24 @@ PluginModelCatalogApi, PluginUploadFileRequestApi, ) +from core.app.apps import base_app_queue_manager from core.datasource.entities.datasource_entities import DatasourceProviderType -from core.plugin.entities.request import RequestInvokeDatasource, RequestInvokeTextEmbedding, RequestListModels +from core.ops.ops_trace_manager import TraceQueueManager +from core.plugin.entities.request import ( + RequestInvokeDatasource, + RequestInvokeTextEmbedding, + RequestInvokeTool, + RequestListModels, +) from core.workflow.file_reference import build_file_reference +from extensions.ext_application_services import ApplicationServices from graphon.model_runtime.entities.model_entities import ModelType from models import Account, Tenant +from models.account import TenantAccountJoin, TenantAccountRole +from models.model import App, AppMode +from models.tools import WorkflowToolProvider +from models.workflow import Workflow, WorkflowType +from services.workflow.execution.adapters.workflow import app_generator as generator_module from tests.helpers.unit.config_override import apply_config_overrides from tests.helpers.unit.model_factories import make_tenant @@ -83,6 +97,93 @@ def _extract_raw_post(cls): return bottom +def test_plugin_invokes_persisted_workflow_tool_through_composed_runtime( + app: Flask, + sqlite_session: Session, + workflow_application: ApplicationServices, + monkeypatch: pytest.MonkeyPatch, +) -> None: + tenant, user = _tenant(), _user() + application = App( + id="nested-app", + tenant_id=tenant.id, + name="Nested", + description="", + mode=AppMode.WORKFLOW, + enable_site=False, + enable_api=True, + max_active_requests=None, + ) + workflow = Workflow.new( + tenant_id=tenant.id, + app_id=application.id, + type=WorkflowType.WORKFLOW.value, + version="1", + graph=json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ), + features="{}", + created_by=user.id, + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) + provider = WorkflowToolProvider( + name="nested", + label="Nested", + icon="icon.svg", + app_id=application.id, + version="1", + user_id=user.id, + tenant_id=tenant.id, + description="", + parameter_configuration="[]", + ) + sqlite_session.add_all( + [ + tenant, + user, + application, + workflow, + provider, + TenantAccountJoin(tenant_id=tenant.id, account_id=user.id, role=TenantAccountRole.OWNER), + ] + ) + sqlite_session.commit() + sqlite_session.close() + # Any fallback to the global database instead of the composed repositories must fail. + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + trace_queue_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + monkeypatch.setattr(generator_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) + monkeypatch.setattr(generator_module.WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + handled = [] + + def handle_response(self, **_kwargs): + assert self._runtime is workflow_application.workflow_runtime + handled.append("workflow response") + return {"data": {"outputs": {"result": "nested result"}}} + + monkeypatch.setattr(generator_module.WorkflowAppGenerator, "_handle_response", handle_response) + monkeypatch.setattr( + generator_module.WorkflowAppGenerateResponseConverter, "convert", lambda *, response, **_kwargs: response + ) + with app.test_request_context(): + response = _extract_raw_post(PluginInvokeToolApi)( + PluginInvokeToolApi(), + user_model=user, + tenant_model=tenant, + payload=RequestInvokeTool( + tool_type="workflow", provider=provider.id, tool=provider.name, tool_parameters={} + ), + ) + messages = [json.loads(frame[14:]) for frame in response.response] + assert handled == ["workflow response"] + assert messages + assert all(message["error"] == "" for message in messages) + assert any("nested result" in json.dumps(message["data"]) for message in messages) + + class TestPluginInvokeLLMApi: """Test PluginInvokeLLMApi endpoint structure""" diff --git a/api/tests/unit_tests/controllers/mcp/test_mcp.py b/api/tests/unit_tests/controllers/mcp/test_mcp.py index 451f61b7751452..553a8d4ed5c19e 100644 --- a/api/tests/unit_tests/controllers/mcp/test_mcp.py +++ b/api/tests/unit_tests/controllers/mcp/test_mcp.py @@ -810,7 +810,7 @@ def test_tools_list_json_unchanged_for_legacy_client(self, app): tool = response.get_json()["result"]["tools"][0] assert set(tool) == {"name", "description", "inputSchema"} - @patch("core.mcp.server.streamable_http.AppGenerateService") + @patch("controllers.mcp.mcp.AppGenerateService") def test_tools_call_json_includes_structured_content_for_modern_client(self, mock_app_generate, app): """A 2025-06-18 client receives structuredContent alongside the text content.""" api = self._make_api() @@ -827,7 +827,7 @@ def test_tools_call_json_includes_structured_content_for_modern_client(self, moc assert result["structuredContent"] == {"answer": "test answer"} assert result["content"][0]["text"] == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") + @patch("controllers.mcp.mcp.AppGenerateService") def test_tools_call_json_omits_structured_content_for_legacy_client(self, mock_app_generate, app): """A 2024-11-05 client receives the pre-upgrade tools/call JSON without structuredContent.""" api = self._make_api() @@ -1062,3 +1062,6 @@ def test_get_mcp_server_and_app_returns_persisted_models(sqlite_session: Session assert result_server is server assert result_app is app + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py index 8bbc876a89e916..1eaeafc2af09a0 100644 --- a/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py +++ b/api/tests/unit_tests/controllers/openapi/auth/test_resource_access.py @@ -10,6 +10,7 @@ from sqlalchemy.orm import Session from werkzeug.exceptions import Forbidden, Unauthorized +from controllers.common.errors import NotFoundError from controllers.openapi._catalog import CATALOG_HEADER, catalog_for from controllers.openapi.auth.context import Context from controllers.openapi.auth.requirements import CheckScope, CheckSubject @@ -259,8 +260,6 @@ def test_stop_task_rejects_other_token_owner( ) -> None: import inspect - from werkzeug.exceptions import NotFound - from controllers.openapi.app_run import AppTaskStopApi flask_app, app_id, _, _, end_user = resource_fixture @@ -274,8 +273,10 @@ def test_stop_task_rejects_other_token_owner( "controllers.openapi.app_run.application_services", return_value=replace(account_application_services, app_tasks=AppTaskControlService(redis_client=redis)), ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError, match="Task not found") as raised: inspect.unwrap(AppTaskStopApi.post)(AppTaskStopApi(), data, app_id, "task") + assert raised.value.code == 404 + assert raised.value.error_code == "not_found" assert redis.operations == [] redis.values["generate_task_belong:task"] = b"end-user-machine-user" inspect.unwrap(AppTaskStopApi.post)(AppTaskStopApi(), data, app_id, "task") diff --git a/api/tests/unit_tests/controllers/openapi/test_app_release.py b/api/tests/unit_tests/controllers/openapi/test_app_release.py index 368ab8bf342e83..ea8b6edfc729e4 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_release.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_release.py @@ -1,28 +1,37 @@ +import json from datetime import datetime, timedelta import pytest from controllers.openapi.app_release import passed_on_draft from graphon.enums import WorkflowExecutionStatus -from models.workflow import Workflow, WorkflowRun, WorkflowType +from models.workflow import WorkflowRun +from services.workflow.contracts import WorkflowRecord _GRAPH = '{"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []}' _OTHER_GRAPH = '{"nodes": [{"id": "llm", "data": {"type": "llm"}}], "edges": []}' _RUN_STARTED = datetime(2026, 10, 15, 12, 0, 0) -def _draft(*, saved_at: datetime, graph: str = _GRAPH) -> Workflow: - draft = Workflow( - tenant_id="workspace-1", - app_id="app-1", - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=graph, - _features="{}", - created_by="account-1", +def _draft(*, saved_at: datetime, graph: str = _GRAPH) -> WorkflowRecord: + return WorkflowRecord( + id="draft-1", + graph=json.loads(graph), + features={}, + hash="draft-hash", + version="draft", + version_number=None, + marked_name="", + marked_comment="", + created_by=None, + created_at=saved_at, + updated_by=None, + updated_at=saved_at, + tool_published=False, + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], ) - draft.updated_at = saved_at - return draft def _run(*, status: WorkflowExecutionStatus = WorkflowExecutionStatus.SUCCEEDED, graph: str = _GRAPH) -> WorkflowRun: diff --git a/api/tests/unit_tests/controllers/openapi/test_app_run.py b/api/tests/unit_tests/controllers/openapi/test_app_run.py index 54a7523b680cea..5d749de15a18a4 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_run.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_run.py @@ -14,9 +14,9 @@ from sqlalchemy import event from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import BadRequest, NotFound, TooManyRequests from constants.oauth_bearer import TokenType +from controllers.common.errors import InvalidRequestError, NotFoundError, TooManyRequestsError from controllers.openapi import app_run from controllers.openapi._errors import AppModeMismatch from controllers.openapi._models import ( @@ -61,7 +61,7 @@ def test_translate_maps_app_concurrency_to_too_many_requests(): # Regression guard: this used to fall through to a 500 (it was not caught here). - with pytest.raises(TooManyRequests) as exc: + with pytest.raises(TooManyRequestsError) as exc: with _translate_service_errors(): raise AppInvokeQuotaExceededError("internal: client_id=abc max=10") assert exc.value.code == 429 @@ -86,7 +86,7 @@ def test_translate_maps_trigger_workflow_to_stable_unavailable_error(): def test_translate_maps_value_error_to_a_fixed_bad_request(): # Regression guard: an unpublished workflow used to surface as a detail-less 500. # The service message names internals, so the client gets a fixed text. - with pytest.raises(BadRequest) as exc: + with pytest.raises(InvalidRequestError) as exc: with _translate_service_errors(): raise ValueError("variable 'secret_key' of node 42 is missing") assert exc.value.code == 400 @@ -191,7 +191,7 @@ def test_stop_task_enforces_resolved_caller_ownership( if allowed: assert api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id) == TaskStopResponse(result="success") else: - with pytest.raises(NotFound, match="Task not found"): + with pytest.raises(NotFoundError, match="Task not found"): api.post.__handler__(api, ctx, app_id="app-1", task_id=task_id) if allowed: assert redis.values[f"generate_task_stopped:{task_id}"] == b"1" @@ -344,6 +344,9 @@ def test_chat_route_passes_a_message_end_without_conversation_through(app: Flask assert list(response.response) == [end] +pytestmark = pytest.mark.usefixtures("workflow_application") + + def test_draft_runs_do_not_hint_the_form_submit_op() -> None: """A draft run's human-input form is a debugger form; the submit op only takes web-app forms, so hinting it would send the agent straight into a 403.""" diff --git a/api/tests/unit_tests/controllers/openapi/test_app_workflow.py b/api/tests/unit_tests/controllers/openapi/test_app_workflow.py index 0daeb7caf7a481..ffa5be9fcafddf 100644 --- a/api/tests/unit_tests/controllers/openapi/test_app_workflow.py +++ b/api/tests/unit_tests/controllers/openapi/test_app_workflow.py @@ -9,7 +9,7 @@ from flask import Flask from pydantic import ValidationError from sqlalchemy import select -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage from werkzeug.exceptions import BadRequest @@ -55,8 +55,9 @@ from models.enums import AppStatus from models.model import App, AppMode, IconType from models.workflow import Workflow, WorkflowType +from services.workflow.contracts import WorkflowPublication, WorkflowRecord from services.workflow.graph_diff import draft_token -from services.workflow_service import WorkflowService +from services.workflow_variable_reference_validator import advisory_variable_reference_warning WORKSPACE_ID = "00000000-0000-4000-8000-000000000001" APP_ID = "00000000-0000-4000-8000-000000000002" @@ -111,12 +112,10 @@ def _app_context(session: Session, **view_args: str) -> Context: return ctx -def _workflow_service(monkeypatch: pytest.MonkeyPatch, session_factory: sessionmaker[Session]) -> WorkflowService: - """`WorkflowService()` builds its sessions from db.engine, which the bare Flask app these tests run - under does not have; hand the handlers the real service over the test database instead.""" - service = WorkflowService(session_factory) - monkeypatch.setattr(app_workflow, "WorkflowService", lambda: service) - return service +def _workflow_services(monkeypatch: pytest.MonkeyPatch, services: ApplicationServices) -> ApplicationServices: + """Inject the real application boundary backed by the test database.""" + monkeypatch.setattr(app_workflow, "application_services", lambda: services) + return services def _workflow( @@ -136,6 +135,29 @@ def _workflow( ) +def _workflow_record(workflow: Workflow) -> WorkflowRecord: + created_at = workflow.created_at or datetime(2026, 10, 1, 12, 0, 0) + updated_at = workflow.updated_at or created_at + return WorkflowRecord( + id=workflow.id, + graph=dict(workflow.graph_dict), + features=workflow.features_dict, + hash=workflow.unique_hash, + version=workflow.version, + version_number=workflow.version_number, + marked_name=workflow.marked_name or "", + marked_comment=workflow.marked_comment or "", + created_by=None, + created_at=created_at, + updated_by=None, + updated_at=updated_at, + tool_published=False, + environment_variables=list(workflow.environment_variables), + conversation_variables=list(workflow.conversation_variables), + rag_pipeline_variables=workflow.rag_pipeline_variables, + ) + + def _add_draft(session: Session, *, graph: str = "{}", environment_variables: Sequence[VariableBase] = ()) -> Workflow: draft = _workflow(graph=graph, environment_variables=environment_variables) session.add(draft) @@ -170,11 +192,11 @@ def test_set_env_rejects_mask_for_unknown_id( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: """Sending the mask for an id that is not a stored secret must not encrypt the literal mask into the draft (the bug: it silently succeeded, storing `[__HIDDEN__]`).""" - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="unknown-id") draft = _add_draft(sqlite_session) @@ -191,11 +213,11 @@ def test_set_env_accepts_mask_for_a_stored_secret( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: """The same mask for an id that *is* a stored secret keeps the existing flow working.""" stored = SecretVariable(id="s-1", name="KEY", value="plain", selector=["env", "KEY"]) - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="s-1") draft = _add_draft(sqlite_session, environment_variables=[stored]) @@ -204,6 +226,7 @@ def test_set_env_accepts_mask_for_a_stored_secret( with app.test_request_context("/openapi/v1/apps/app-1/env/s-1", method="PUT"): cast(_EndpointView, api.put).__handler__(api, ctx, "app-1", "s-1", body=body) + sqlite_session.refresh(draft) assert _env(draft) == [("s-1", "plain", SecretVariable)] assert draft.updated_by == ACCOUNT_ID @@ -212,12 +235,12 @@ def test_set_env_rejects_the_mask_as_a_plain_value( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: """Re-typing a stored secret as a string while sending the mask would store the mask text and destroy the secret.""" stored = SecretVariable(id="s-1", name="KEY", value="plain", selector=["env", "KEY"]) - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="s-1") draft = _add_draft(sqlite_session, environment_variables=[stored]) @@ -234,9 +257,9 @@ def test_set_env_without_a_draft_is_not_found( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="e-1") body = EnvVariableSetPayload(name="KEY", value_type="string", value="v") @@ -252,9 +275,9 @@ def test_delete_env_without_a_draft_is_not_found( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="e-1") api = AppEnvItemApi() @@ -269,11 +292,11 @@ def test_delete_env_with_an_unknown_id_is_not_found( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: """An unknown id must not report success while deleting nothing.""" stored = StringVariable(id="e-1", name="KEY", value="v", selector=["env", "KEY"]) - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="typo") draft = _add_draft(sqlite_session, environment_variables=[stored]) @@ -289,11 +312,11 @@ def test_delete_env_removes_a_stored_variable( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: stored = StringVariable(id="e-1", name="KEY", value="v", selector=["env", "KEY"]) kept = StringVariable(id="e-2", name="URL", value="https://x", selector=["env", "URL"]) - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session, env_id="e-1") draft = _add_draft(sqlite_session, environment_variables=[stored, kept]) @@ -301,6 +324,7 @@ def test_delete_env_removes_a_stored_variable( with app.test_request_context("/openapi/v1/apps/app-1/env/e-1", method="DELETE"): cast(_EndpointView, api.delete).__handler__(api, ctx, "app-1", "e-1") + sqlite_session.refresh(draft) assert _env(draft) == [("e-2", "https://x", StringVariable)] @@ -321,15 +345,16 @@ def test_version_list_excludes_the_draft( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - _workflow_service(monkeypatch, sqlite_session_factory) + _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session) _add_draft(sqlite_session) published = _workflow(version=Workflow.version_from_datetime(datetime(2026, 10, 1, 12, 0, 0))) sqlite_session.add(published) sqlite_session.commit() ctx.app.workflow_id = published.id + sqlite_session.commit() api = AppVersionListApi() with app.test_request_context("/openapi/v1/apps/app-1/versions"): @@ -342,11 +367,12 @@ def test_restore_returns_the_hash_an_import_checks( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) + services = _workflow_services(monkeypatch, account_application_services) restored = _workflow(graph=_branch_skipping_graph()) - monkeypatch.setattr(service, "restore_published_workflow_to_draft", lambda **_kwargs: restored) + monkeypatch.setattr(services.console_workflows, "restore", lambda *_args: None) + monkeypatch.setattr(services.console_workflows, "draft", lambda *_args: _workflow_record(restored)) version_id = "11111111-1111-4111-8111-111111111111" ctx = _app_context(sqlite_session, version_id=version_id) @@ -398,9 +424,11 @@ def _upload_part() -> FileStorage: def test_run_draft_node_merges_local_files_into_the_inputs( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] + monkeypatch: pytest.MonkeyPatch, + sqlite_session: Session, + account_application_services: ApplicationServices, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) + services = _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session) _add_draft(sqlite_session, graph=_code_node_graph("n1")) merges: list[tuple[object, ...]] = [] @@ -410,15 +438,19 @@ def merge_files(inputs: object, files: object, caller: object) -> dict[str, obje return {"x": 1, "doc": {"id": "file-1"}} node_runs: list[dict[str, object]] = [] + + def run_node(_context: object, app_id: str, node_id: str, args: dict[str, object]) -> dict[str, object]: + node_runs.append({"app_id": app_id, "node_id": node_id, **args}) + return {"id": "e-1"} + monkeypatch.setattr(app_workflow, "merge_files", merge_files) - monkeypatch.setattr(service, "run_draft_workflow_node", lambda **kwargs: node_runs.append(kwargs)) - monkeypatch.setattr(app_workflow, "node_execution_response_source", lambda *_args, **_kwargs: {"id": "e-1"}) + monkeypatch.setattr(services.console_workflows, "run_node", run_node) parts: UploadParts = {"doc": _upload_part()} run_draft_node(ctx, "n1", inputs={"x": 1}, query="", files=parts) assert merges == [({"x": 1}, parts, ctx.caller)] - assert [run["user_inputs"] for run in node_runs] == [{"x": 1, "doc": {"id": "file-1"}}] + assert [run["inputs"] for run in node_runs] == [{"x": 1, "doc": {"id": "file-1"}}] def test_node_run_payloads_take_files_as_multipart_parts() -> None: @@ -430,11 +462,11 @@ def test_node_run_payloads_take_files_as_multipart_parts() -> None: def test_run_draft_node_ends_the_read_transaction_before_uploading( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, with_files: bool, ends_transaction: bool, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) + services = _workflow_services(monkeypatch, account_application_services) ctx = _app_context(sqlite_session) _add_draft(sqlite_session, graph=_code_node_graph("n1")) names: list[str] = [] @@ -445,8 +477,7 @@ def merge_files(inputs: dict[str, object], *_args: object) -> dict[str, object]: return inputs monkeypatch.setattr(app_workflow, "merge_files", merge_files) - monkeypatch.setattr(service, "run_draft_workflow_node", lambda **_kwargs: None) - monkeypatch.setattr(app_workflow, "node_execution_response_source", lambda *_args, **_kwargs: {"id": "e-1"}) + monkeypatch.setattr(services.console_workflows, "run_node", lambda *_args: {"id": "e-1"}) files: UploadParts | None = {"doc": _upload_part()} if with_files else None run_draft_node(ctx, "n1", inputs={}, query="", files=files) @@ -480,32 +511,34 @@ def _published_version(graph: str) -> Workflow: @pytest.mark.parametrize("graph", ["{}", "{", ""], ids=["clean", "invalid-json", "empty"]) -def test_publish_points_the_app_at_the_new_version( +def test_publish_returns_the_new_active_version( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, graph: str, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) - publishes: list[dict[str, object]] = [] + services = _workflow_services(monkeypatch, account_application_services) + publishes: list[tuple[tuple[object, ...], dict[str, object]]] = [] - def publish_workflow(**kwargs: object) -> Workflow: - publishes.append(kwargs) - return _published_version(graph) + def publish_workflow(*args: object, **kwargs: object) -> tuple[WorkflowPublication, str | None]: + publishes.append((args, kwargs)) + return ( + WorkflowPublication(datetime(2026, 10, 1, 12, 0, 0), graph), + advisory_variable_reference_warning(graph), + ) - monkeypatch.setattr(service, "publish_workflow", publish_workflow) + monkeypatch.setattr(services.console_workflows, "publish", publish_workflow) + monkeypatch.setattr( + services.console_workflows, "published", lambda *_args: _workflow_record(_published_version("{}")) + ) ctx = _app_context(sqlite_session) api = AppPublishApi() with app.test_request_context("/openapi/v1/apps/app-1:publish", method="POST"): response = cast(_EndpointView, api.post).__handler__(api, ctx, "app-1", body=PublishPayload(marked_name="v2")) - stored_app = sqlite_session.get(App, APP_ID) - assert stored_app is not None - assert [publish["marked_name"] for publish in publishes] == ["v2"] - assert (stored_app.workflow_id, stored_app.updated_by) == (VERSION_ID, ACCOUNT_ID) - assert stored_app.updated_at is not None + assert [(args[1], kwargs["marked_name"]) for args, kwargs in publishes] == [(APP_ID, "v2")] assert cast(PublishResponse, response).version_id == VERSION_ID assert cast(PublishResponse, response).warning is None @@ -514,10 +547,21 @@ def test_publish_warns_about_references_to_a_skipped_branch( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) - monkeypatch.setattr(service, "publish_workflow", lambda **_kwargs: _published_version(_branch_skipping_graph())) + services = _workflow_services(monkeypatch, account_application_services) + graph = _branch_skipping_graph() + monkeypatch.setattr( + services.console_workflows, + "publish", + lambda *_args, **_kwargs: ( + WorkflowPublication(datetime(2026, 10, 1, 12, 0, 0), graph), + advisory_variable_reference_warning(graph), + ), + ) + monkeypatch.setattr( + services.console_workflows, "published", lambda *_args: _workflow_record(_published_version("{}")) + ) ctx = _app_context(sqlite_session) api = AppPublishApi() @@ -534,14 +578,14 @@ def test_publish_refusal_is_a_bad_request( app: Flask, monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, - sqlite_session_factory: sessionmaker[Session], + account_application_services: ApplicationServices, ) -> None: - service = _workflow_service(monkeypatch, sqlite_session_factory) + services = _workflow_services(monkeypatch, account_application_services) - def publish_workflow(**_kwargs: object) -> Workflow: + def publish_workflow(*_args: object, **_kwargs: object) -> tuple[WorkflowPublication, str | None]: raise ValueError("Node l (llm): Missing provider or model configuration") - monkeypatch.setattr(service, "publish_workflow", publish_workflow) + monkeypatch.setattr(services.console_workflows, "publish", publish_workflow) ctx = _app_context(sqlite_session) api = AppPublishApi() diff --git a/api/tests/unit_tests/controllers/service_api/app/test_completion.py b/api/tests/unit_tests/controllers/service_api/app/test_completion.py index 700b6b58512b3c..ba111537f3cef7 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_completion.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_completion.py @@ -10,9 +10,10 @@ from pytest_mock import MockerFixture from sqlalchemy import Engine from sqlalchemy.orm import Session -from werkzeug.exceptions import BadRequest, NotFound +from werkzeug.exceptions import BadRequest import services +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.service_api.app import completion as completion_module from controllers.service_api.app import workflow as workflow_module from controllers.service_api.app.completion import ( @@ -32,7 +33,6 @@ WorkflowVersionExecutionNotAllowedError, ) from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.errors.error import QuotaExceededError from core.helper.trace_id_helper import get_trace_session_id from enums import CloudPlan, DeploymentEdition @@ -41,6 +41,7 @@ from models.base import TypeBase from models.enums import ConversationFromSource, EndUserType from models.model import App, AppMode, Conversation, EndUser, IconType, Message +from services.app.generation.errors import AgentAppNotPublishedError from services.app_generate_service import AppGenerateService from services.app_task_service import AppTaskService from services.billing_service import BillingService @@ -48,6 +49,8 @@ from services.errors.app import IsDraftWorkflowError, WorkflowIdFormatError, WorkflowNotFoundError from services.errors.conversation import ConversationNotExistsError from services.errors.llm import InvokeRateLimitError +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers import billing_builders as builders from tests.helpers.unit.config_override import apply_config_overrides from tests.helpers.unit.model_factories import make_app, make_end_user @@ -347,7 +350,14 @@ def test_generate_method_exists(self): assert callable(AppGenerateService.generate) @patch.object(AppGenerateService, "generate") - def test_generate_returns_response(self, mock_generate, orm_session: Session): + def test_generate_returns_response( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test that generate returns expected response format.""" expected = {"answer": "Hello!"} mock_generate.return_value = expected @@ -360,12 +370,21 @@ def test_generate_returns_response(self, mock_generate, orm_session: Session): invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == expected @patch.object(AppGenerateService, "generate") - def test_generate_raises_conversation_not_exists(self, mock_generate, orm_session: Session): + def test_generate_raises_conversation_not_exists( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises ConversationNotExistsError.""" mock_generate.side_effect = services.errors.conversation.ConversationNotExistsError() app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -378,10 +397,19 @@ def test_generate_raises_conversation_not_exists(self, mock_generate, orm_sessio invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) @patch.object(AppGenerateService, "generate") - def test_generate_raises_quota_exceeded(self, mock_generate, orm_session: Session): + def test_generate_raises_quota_exceeded( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises QuotaExceededError.""" mock_generate.side_effect = QuotaExceededError() app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -394,10 +422,19 @@ def test_generate_raises_quota_exceeded(self, mock_generate, orm_session: Sessio invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) @patch.object(AppGenerateService, "generate") - def test_generate_raises_invoke_error(self, mock_generate, orm_session: Session): + def test_generate_raises_invoke_error( + self, + mock_generate, + orm_session: Session, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowRuntime, + ): """Test generate raises InvokeError.""" mock_generate.side_effect = InvokeError("Model invocation failed") app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.COMPLETION) @@ -410,6 +447,8 @@ def test_generate_raises_invoke_error(self, mock_generate, orm_session: Session) invoke_from=Mock(), session=orm_session, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) class TestCompletionControllerLogic: @@ -556,7 +595,7 @@ def test_conversation_not_found( handler = unwrap(api.post) with app.test_request_context("/completion-messages", method="POST", json={"inputs": {}}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) class TestCompletionStopApiController: @@ -719,7 +758,7 @@ def test_workflow_not_found(self, app: Flask, monkeypatch: pytest.MonkeyPatch, o app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.CHAT) with app.test_request_context("/chat-messages", method="POST", json={"inputs": {}, "query": "hi"}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) def test_draft_workflow(self, app: Flask, monkeypatch: pytest.MonkeyPatch, orm_session: Session) -> None: @@ -734,7 +773,7 @@ def test_draft_workflow(self, app: Flask, monkeypatch: pytest.MonkeyPatch, orm_s app_model, end_user, _, _ = _persist_completion_state(orm_session, AppMode.CHAT) with app.test_request_context("/chat-messages", method="POST", json={"inputs": {}, "query": "hi"}): - with pytest.raises(BadRequest): + with pytest.raises(InvalidRequestError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) def test_agent_not_published_error_mapped( @@ -777,7 +816,7 @@ def test_invalid_conversation_id_fails_fast_as_not_found( method="POST", json={"inputs": {}, "query": "hi", "conversation_id": str(uuid.uuid4())}, ): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=orm_session, app_model=app_model, end_user=end_user) # The lookup must run before generation, so the generator is never started. @@ -964,3 +1003,6 @@ def test_workflow_run_api_passes_body_trace_session_id( assert response == {"result": "ok"} _assert_generate_trace_session_id(mock_generate_service, "body-session") + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py b/api/tests/unit_tests/controllers/service_api/app/test_workflow.py index f27ab593aa6827..10dfed3fc66584 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_workflow.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_workflow.py @@ -13,8 +13,8 @@ from pytest_mock import MockerFixture from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker -from werkzeug.exceptions import BadRequest, NotFound +from controllers.common.errors import InvalidRequestError, NotFoundError from controllers.service_api.app.error import ( CompletionRequestError, NotWorkflowAppError, @@ -40,7 +40,7 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.model import App, AppMode, EndUser from models.workflow import WorkflowAppLog, WorkflowAppLogCreatedFrom, WorkflowRun, WorkflowType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository from services.app_generate_service import AppGenerateService from services.billing_client import transport from services.billing_service import BillingService @@ -53,7 +53,7 @@ ) from services.errors.billing import BillingUpstreamInvalidResponseError from services.errors.llm import InvokeRateLimitError -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from services.workflow.app_log_query_service import WorkflowAppLogQueryService from tests.helpers import billing_builders as builders """ @@ -721,7 +721,7 @@ def test_not_found( end_user = _make_end_user() with app.test_request_context("/workflows/1/run", method="POST", json={"inputs": {}}): - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): handler(api, session=sqlite_session, app_model=app_model, end_user=end_user, workflow_id="w1") @pytest.mark.parametrize("sqlite_session", [()], indirect=True) @@ -746,7 +746,7 @@ def test_draft_workflow( end_user = _make_end_user() with app.test_request_context("/workflows/1/run", method="POST", json={"inputs": {}}): - with pytest.raises(BadRequest): + with pytest.raises(InvalidRequestError): handler(api, session=sqlite_session, app_model=app_model, end_user=end_user, workflow_id="w1") class TestWorkflowTaskStopApi: @@ -797,7 +797,7 @@ def test_success( _persist_workflow_log(sqlite_session, tenant_id=app_model.tenant_id, app_id=app_model.id) workflow_app_logs = WorkflowAppLogQueryService( - logs=WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory), + logs=WorkflowAppLogRepository(session_factory=sqlite_session_factory), ) monkeypatch.setattr( sys.modules["controllers.service_api.app.workflow"], @@ -940,3 +940,6 @@ def test_workflow_run_serializer_running_returns_outputs() -> None: response = dump_response(WorkflowRunResponse, _workflow_run(WorkflowExecutionStatus.RUNNING)) assert response["outputs"] == {"foo": "bar"} + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py b/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py index d8ef7f41d61497..8a139a73689442 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_workflow_events.py @@ -53,8 +53,10 @@ from repositories.api_workflow_node_execution_repository import WorkflowNodeExecutionSnapshot from repositories.entities.workflow_pause import WorkflowPauseEntity from services.app.generation.ports import ConversationSnapshot, MessageSnapshot, WorkflowSnapshot +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_event_snapshot_service import _build_paused_snapshot_events from tests.helpers.unit.config_override import apply_config_overrides @@ -438,6 +440,9 @@ def test_advanced_chat_blocking_injects_pause_state_config( self, monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, ) -> None: apply_config_overrides(monkeypatch, DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) monkeypatch.setattr(ags_module, "RateLimit", _DummyRateLimit) @@ -449,9 +454,9 @@ def test_advanced_chat_blocking_injects_pause_state_config( generator_instance = Mock() generator_instance.generate.return_value = {"result": "advanced-blocking"} - generator_instance.convert_to_event_stream.side_effect = lambda payload: payload generator_factory = Mock(return_value=generator_instance) monkeypatch.setattr(ags_module, "AdvancedChatAppGenerator", generator_factory) + monkeypatch.setattr(ags_module, "convert_to_event_stream", lambda payload: payload) app_model = _hitl_service_api_app(app_id="app-id", tenant_id="tenant-id", mode=AppMode.ADVANCED_CHAT) user = _hitl_service_api_end_user(user_id="user-id", app_id="app-id", tenant_id="tenant-id") @@ -464,10 +469,17 @@ def test_advanced_chat_blocking_injects_pause_state_config( args={"workflow_id": None, "query": "hi", "inputs": {}}, invoke_from=InvokeFrom.SERVICE_API, streaming=False, + variables=workflow_variables, + runtime=workflow_runtime, ) assert result == {"result": "advanced-blocking"} - generator_factory.assert_called_once_with(execution_driver=ANY) + generator_factory.assert_called_once_with( + runtime=workflow_runtime, + execution_driver=workflow_runtime.execution_driver, + draft_variable_loader=workflow_variables.workflow_loader, + draft_variable_saver=workflow_variables.saver_factory, + ) call_kwargs = generator_instance.generate.call_args.kwargs assert call_kwargs["streaming"] is False assert call_kwargs["pause_state_config"] is not None @@ -608,10 +620,7 @@ def _gen(): assert response.data.reasons[0]["form_id"] == "form-1" assert response.data.reasons[0]["expiration_time"] == 123 - def test_workflow_blocking_pipeline_pause_payload_contract( - self, monkeypatch: pytest.MonkeyPatch, *, workflow_runtime, tool_providers - ) -> None: - from services.workflow.execution.adapters.workflow import generate_task_pipeline as workflow_pipeline_module + def test_workflow_blocking_pipeline_pause_payload_contract(self, *, workflow_runtime, tool_providers) -> None: from services.workflow.execution.adapters.workflow.generate_task_pipeline import WorkflowAppGenerateTaskPipeline app_config = WorkflowUIBasedAppConfig( @@ -646,7 +655,6 @@ def test_workflow_blocking_pipeline_pause_payload_contract( draft_var_saver_factory=lambda **kwargs: None, tool_providers=tool_providers, ) - monkeypatch.setattr(workflow_pipeline_module.time, "time", lambda: 1700000000) def _gen(): yield HumanInputRequiredResponse( diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py index 4d78dd4e98a32e..506d43f43cb740 100644 --- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py +++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py @@ -10,11 +10,12 @@ from flask import Flask from sqlalchemy.orm import Session, scoped_session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import Forbidden, NotFound from controllers.common.errors import ( + AccessDeniedError, FilenameNotExistsError, NoFileUploadedError, + NotFoundError, TooManyFilesError, ) from controllers.common.errors import ( @@ -33,16 +34,14 @@ PipelineRunApi, PipelineUploadFileResponse, ) -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom +from extensions.ext_application_services import ApplicationServices from extensions.storage.storage_type import StorageType from libs.helper import dump_response from models.account import Account from models.dataset import Dataset, Pipeline from models.enums import CreatorUserRole from models.model import UploadFile -from repositories.credentials.query_repository import CredentialQueryRepository -from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository from services.errors.file import FileTooLargeError as FileTooLargeServiceError from services.errors.file import UnsupportedFileTypeError from services.rag_pipeline.entity.pipeline_service_api_entities import ( @@ -50,7 +49,7 @@ PipelineRunApiEntity, ) from services.rag_pipeline.rag_pipeline import RagPipelineService -from tests.helpers.unit.credential_refresh import build_datasource_provider_service +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from tests.helpers.unit.model_factories import make_upload_file """ @@ -97,18 +96,14 @@ def _persist_pipeline(session: Session, *, tenant_id: str) -> Pipeline: class TestRagPipelineWorkflowCases: @staticmethod @pytest.fixture(autouse=True) - def _bind_database(sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch): + def _bind_database( + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + workflow_application: ApplicationServices, + ): session_proxy = scoped_session(sqlite_session_factory) monkeypatch.setattr(workflow_module.db, "session", session_proxy) - query = CredentialQueryRepository(session_factory=sqlite_session_factory) - providers = build_datasource_provider_service( - credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory) - ) - monkeypatch.setattr( - workflow_module, - "application_services", - lambda: SimpleNamespace(credential_queries=query, data_sources=SimpleNamespace(providers=providers)), - ) + monkeypatch.setattr(workflow_module, "application_services", lambda: workflow_application) yield session_proxy.remove() @@ -210,10 +205,6 @@ def test_get_pipeline_method_exists(self): """Test RagPipelineService.get_pipeline exists.""" assert hasattr(RagPipelineService, "get_pipeline") - def test_run_datasource_workflow_node_method_exists(self): - """Test RagPipelineService.run_datasource_workflow_node exists.""" - assert hasattr(RagPipelineService, "run_datasource_workflow_node") - def test_get_pipeline_templates_method_exists(self): """Test RagPipelineService.get_pipeline_templates exists.""" assert hasattr(RagPipelineService, "get_pipeline_templates") @@ -493,14 +484,14 @@ def test_get_plugins_parses_false_is_published_query(self, mock_svc_cls, app: Fl ) def test_get_plugins_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" tenant_id = str(uuid.uuid4()) dataset_id = str(uuid.uuid4()) _persist_dataset(sqlite_session, tenant_id="other-tenant", dataset_id=dataset_id) with app.test_request_context("/datasets/test/pipeline/datasource-plugins"): api = DatasourcePluginsApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.get(tenant_id=tenant_id, dataset_id=dataset_id) @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.RagPipelineService") @@ -524,7 +515,7 @@ class TestDatasourceNodeRunApiPost: """Tests for DatasourceNodeRunApi.post(). The source asserts ``isinstance(current_user, Account)`` and delegates to - ``RagPipelineService`` and ``PipelineGenerator``, so we patch those plus + ``RagPipelineService`` and the event-stream adapter, so we patch those plus ``current_user``. ``post`` is wrapped in ``@model_validate``, which parses the JSON request body live, so payloads are supplied via ``test_request_context(json=...)`` and validation runs before the dataset @@ -532,7 +523,7 @@ class TestDatasourceNodeRunApiPost: """ @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.helper") - @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.PipelineGenerator") + @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.convert_to_event_stream") @patch( "controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.current_user", new_callable=lambda: Account(name="Test Account", email="test@example.com"), @@ -557,41 +548,46 @@ def test_post_success( pipeline = _persist_pipeline(sqlite_session, tenant_id=tenant_id) mock_svc_instance = Mock() mock_svc_instance.get_pipeline.return_value = pipeline - mock_svc_instance.run_datasource_workflow_node.return_value = iter(["event1"]) mock_svc_cls.return_value = mock_svc_instance - mock_gen.convert_to_event_stream.return_value = iter(["stream_event"]) + mock_gen.return_value = iter(["stream_event"]) mock_helper.compact_generate_response.return_value = {"result": "ok"} - with app.test_request_context( - "/datasets/test/pipeline/datasource/nodes/node_abc/run", - method="POST", - json={ - "inputs": {"url": "https://example.com"}, - "datasource_type": "online_document", - "is_published": True, - }, + with ( + app.test_request_context( + "/datasets/test/pipeline/datasource/nodes/node_abc/run", + method="POST", + json={ + "inputs": {"url": "https://example.com"}, + "datasource_type": "online_document", + "is_published": True, + }, + ), + patch.object( + workflow_module.application_services().knowledge.pipeline_execution, + "run_datasource_workflow_node", + return_value=iter(["event1"]), + ) as run, ): api = DatasourceNodeRunApi() response = api.post(tenant_id=tenant_id, dataset_id=dataset_id, node_id=node_id) assert response == {"result": "ok"} mock_svc_instance.get_pipeline.assert_called_once_with(tenant_id=tenant_id, dataset_id=dataset_id) - mock_svc_instance.get_pipeline.assert_called_once_with(tenant_id=tenant_id, dataset_id=dataset_id) - mock_svc_instance.run_datasource_workflow_node.assert_called_once() + run.assert_called_once() def test_post_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" # `@model_validate` parses the body before the ownership guard, so a - # valid payload is required to reach the NotFound branch. + # valid payload is required to reach the NotFoundError branch. with app.test_request_context( "/datasets/test/pipeline/datasource/nodes/n1/run", method="POST", json={"inputs": {}, "datasource_type": "online_document", "is_published": True}, ): api = DatasourceNodeRunApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.post(tenant_id=str(uuid.uuid4()), dataset_id=str(uuid.uuid4()), node_id="n1") @patch( @@ -666,13 +662,15 @@ def test_post_success_streaming( mock_svc_cls.assert_called_once_with(sqlite_session) mock_gen_svc.generate.assert_called_once() assert mock_gen_svc.generate.call_args.kwargs["generator"] is pipeline_application - assert mock_gen_svc.generate.call_args.kwargs["session"] is sqlite_session + assert mock_gen_svc.generate.call_args.kwargs["pipeline"] is pipeline + assert mock_gen_svc.generate.call_args.kwargs["invoke_from"] == InvokeFrom.PUBLISHED_PIPELINE + assert mock_gen_svc.generate.call_args.kwargs["streaming"] is True def test_post_not_found(self, app: Flask, sqlite_session: Session): - """Test NotFound when dataset check fails.""" + """Test NotFoundError when dataset check fails.""" with app.test_request_context("/datasets/test/pipeline/run", method="POST"): api = PipelineRunApi() - with pytest.raises(NotFound): + with pytest.raises(NotFoundError): api.post.__wrapped__( api, sqlite_session, @@ -683,7 +681,7 @@ def test_post_not_found(self, app: Flask, sqlite_session: Session): @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.current_user", new="not_account") @patch("controllers.service_api.dataset.rag_pipeline.rag_pipeline_workflow.service_api_ns") def test_post_forbidden_non_account_user(self, mock_ns, app: Flask, sqlite_session: Session): - """Test Forbidden when current_user is not an Account.""" + """Test AccessDeniedError when current_user is not an Account.""" tenant_id = str(uuid.uuid4()) dataset_id = str(uuid.uuid4()) _persist_dataset(sqlite_session, tenant_id=tenant_id, dataset_id=dataset_id) @@ -698,7 +696,7 @@ def test_post_forbidden_non_account_user(self, mock_ns, app: Flask, sqlite_sessi with app.test_request_context("/datasets/test/pipeline/run", method="POST"): api = PipelineRunApi() - with pytest.raises(Forbidden): + with pytest.raises(AccessDeniedError): api.post.__wrapped__( api, sqlite_session, diff --git a/api/tests/unit_tests/controllers/trigger/test_webhook.py b/api/tests/unit_tests/controllers/trigger/test_webhook.py index 8537780e1e2313..d2fc8d90144bcc 100644 --- a/api/tests/unit_tests/controllers/trigger/test_webhook.py +++ b/api/tests/unit_tests/controllers/trigger/test_webhook.py @@ -1,7 +1,9 @@ +from collections.abc import Callable from dataclasses import dataclass from unittest.mock import patch import pytest +from flask import Flask from werkzeug.exceptions import NotFound, RequestEntityTooLarge import controllers.trigger.webhook as module @@ -58,6 +60,42 @@ def _workflow() -> Workflow: return make_workflow(workflow_id="workflow-1") +@pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) +def test_webhook_body_limit_remains_http_413( + monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], path: str +) -> None: + app = Flask(__name__) + config_overrides(WEBHOOK_REQUEST_BODY_MAX_SIZE=8) + monkeypatch.setattr( + module.WebhookService, + "get_webhook_trigger_and_workflow", + lambda *_args, **_kwargs: (_webhook_trigger(), _workflow(), {}), + ) + app.register_blueprint(module.bp) + response = app.test_client().post(f"/triggers/{path}/wh-1", data="long request body") + assert response.status_code == 413 + assert b"Webhook request too large" in response.data + + +@pytest.mark.parametrize("path", ["webhook", "webhook-debug"]) +def test_framework_body_limit_is_not_translated_to_internal_error( + monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], path: str +) -> None: + app = Flask(__name__) + app.config["MAX_CONTENT_LENGTH"] = 8 + config_overrides(WEBHOOK_REQUEST_BODY_MAX_SIZE=1024) + monkeypatch.setattr( + module.WebhookService, + "get_webhook_trigger_and_workflow", + lambda *_args, **_kwargs: (_webhook_trigger(), _workflow(), {}), + ) + app.register_blueprint(module.bp) + + response = app.test_client().post(f"/triggers/{path}/wh-1", json={"message": "request too large"}) + + assert response.status_code == 413 + + class TestPrepareWebhookExecution: @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow") @patch.object(module.WebhookService, "extract_and_validate_webhook_data") @@ -183,10 +221,12 @@ def test_request_entity_too_large(self, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) def test_internal_error(self, mock_get): - response, status = module.handle_webhook("wh-1") + app = Flask(__name__) + app.register_blueprint(module.bp) + response = app.test_client().post("/triggers/webhook/wh-1") - assert status == 500 - assert response["error"] == "Internal server error" + assert response.status_code == 500 + assert response.json == {"error": "Internal server error", "message": "boom"} class TestHandleWebhookDebug: @@ -262,7 +302,9 @@ def test_debug_request_entity_too_large(self, mock_get): @patch.object(module.WebhookService, "get_webhook_trigger_and_workflow", side_effect=Exception("boom")) def test_debug_internal_error(self, mock_get): - response, status = module.handle_webhook_debug("wh-1") + app = Flask(__name__) + app.register_blueprint(module.bp) + response = app.test_client().post("/triggers/webhook-debug/wh-1") - assert status == 500 - assert response["error"] == "Internal server error" + assert response.status_code == 500 + assert response.json == {"error": "Internal server error", "message": "An internal error has occurred."} diff --git a/api/tests/unit_tests/controllers/web/test_completion.py b/api/tests/unit_tests/controllers/web/test_completion.py index 63b400ff813e12..e73e1bdc7574bc 100644 --- a/api/tests/unit_tests/controllers/web/test_completion.py +++ b/api/tests/unit_tests/controllers/web/test_completion.py @@ -9,8 +9,8 @@ import pytest from flask import Flask from sqlalchemy.orm import Session -from werkzeug.exceptions import InternalServerError +from controllers.common.errors import InternalServerError from controllers.web.completion import ChatApi, ChatStopApi, CompletionApi, CompletionStopApi from controllers.web.error import ( AgentNotPublishedError, @@ -21,10 +21,10 @@ ProviderNotInitializeError, ProviderQuotaExceededError, ) -from core.app.apps.agent_app.errors import AgentAppNotPublishedError from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from graphon.model_runtime.errors.invoke import InvokeError from models.model import App, AppMode, EndUser +from services.app.generation.errors import AgentAppNotPublishedError from tests.helpers.unit.model_factories import make_app, make_end_user @@ -236,3 +236,6 @@ def test_stop_success(self, mock_stop: MagicMock, app: Flask) -> None: assert status == 200 assert result == {"result": "success"} + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/controllers/web/test_workflow.py b/api/tests/unit_tests/controllers/web/test_workflow.py index 26740c0ae6cd22..6a283555d461fd 100644 --- a/api/tests/unit_tests/controllers/web/test_workflow.py +++ b/api/tests/unit_tests/controllers/web/test_workflow.py @@ -121,3 +121,6 @@ def test_stop_passes_app_scope(self, stop: MagicMock, app: Flask) -> None: stop.assert_called_once_with( tenant_id="tenant-1", app_id="app-1", task_id="task-1", app_mode=AppMode.WORKFLOW, owner=None ) + + +pytestmark = pytest.mark.usefixtures("workflow_application") diff --git a/api/tests/unit_tests/core/agent/test_base_agent_runner.py b/api/tests/unit_tests/core/agent/test_base_agent_runner.py index dc8fab1fdec28b..121b6aaf0201e0 100644 --- a/api/tests/unit_tests/core/agent/test_base_agent_runner.py +++ b/api/tests/unit_tests/core/agent/test_base_agent_runner.py @@ -22,6 +22,7 @@ from core.app.entities.app_invoke_entities import ( AgentChatAppGenerateEntity, InvokeFrom, + UserFrom, ) from core.model_manager import ModelInstance from core.tools.__base.tool import Tool @@ -722,6 +723,36 @@ def test_convert_tool_to_prompt_message_tool(runner: BaseAgentRunner, mocker: Mo assert prompt_tool.parameters == schema +@pytest.mark.parametrize( + ("invoke_from", "expected_user_from"), + [ + (InvokeFrom.DEBUGGER, UserFrom.ACCOUNT), + (InvokeFrom.SERVICE_API, UserFrom.END_USER), + ], +) +def test_convert_tool_to_prompt_message_tool_attaches_run_context( + runner: BaseAgentRunner, + mocker: MockerFixture, + invoke_from: InvokeFrom, + expected_user_from: UserFrom, +) -> None: + runner.application_generate_entity.invoke_from = invoke_from + tool_entity = _tool("tool1", []) + mocker.patch.object(module.ToolManager, "get_agent_tool_runtime", return_value=tool_entity) + + _, converted = runner._convert_tool_to_prompt_message_tool(_agent_tool("tool1")) + + run_context = converted.runtime.dify_run_context + assert run_context is not None + assert ( + run_context.tenant_id, + run_context.app_id, + run_context.user_id, + run_context.user_from, + run_context.invoke_from, + ) == ("tenant", "app1", "user", expected_user_from, invoke_from) + + def test_convert_dataset_retriever_tool( runner: BaseAgentRunner, mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session] ) -> None: diff --git a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py index 9807bb3806bff4..e697de88f13ccd 100644 --- a/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/agent_app/test_app_generator.py @@ -282,6 +282,32 @@ def test_generate_loads_existing_conversation(self, generator: AgentAppGenerator assert generator._agent_configs.resolve.call_args.kwargs["conversation_id"] == "conv" assert "session" not in generator._init_generate_records.call_args.kwargs + def test_record_initialization_failure_does_not_capture_a_worker_terminal( + self, generator: AgentAppGenerator, mocker: MockerFixture + ) -> None: + generator._prepare_user_inputs = mocker.MagicMock(return_value={}) + generator._init_generate_records = mocker.MagicMock(side_effect=ValueError("record initialization failed")) + mocker.patch( + f"{MODULE}.AgentAppConfigManager.get_app_config", + return_value=mocker.MagicMock(variables=[], tenant_id="tenant", app_id="app1"), + ) + mocker.patch(f"{MODULE}.ModelConfigConverter.convert", return_value=mocker.MagicMock(model="gpt-4o-mini")) + mocker.patch(f"{MODULE}.TraceQueueManager", return_value=mocker.MagicMock()) + mocker.patch(f"{MODULE}.AgentAppGenerateEntity", return_value=mocker.MagicMock()) + thread_constructor = mocker.patch(f"{MODULE}.threading.Thread") + tracing_hooks = mocker.patch("core.ops.base_trace_instance.application_tracing_hooks") + + with pytest.raises(ValueError, match="record initialization failed"): + generator.generate( + app_model=_app(), + user=_account(), + args={"query": "query", "inputs": {}}, + invoke_from=InvokeFrom.DEBUGGER, + ) + + thread_constructor.assert_not_called() + tracing_hooks.assert_not_called() + def test_generate_does_not_include_trace_session_id_in_extras( self, generator: AgentAppGenerator, mocker: MockerFixture ): @@ -467,8 +493,22 @@ def test_generate_task_stopped_is_swallowed(self, generator, mocker: MockerFixtu self._call(generator, mocker, queue_manager) queue_manager.publish_error.assert_not_called() - def test_unexpected_error_is_published(self, generator: AgentAppGenerator, mocker: MockerFixture): - self._wire(generator, mocker, run_side_effect=ValueError("boom")) + @pytest.mark.parametrize("failure_stage", ["record_load", "runner"]) + def test_unexpected_error_is_published_and_captures_failed_terminal( + self, generator: AgentAppGenerator, mocker: MockerFixture, failure_stage: str + ) -> None: + self._wire(generator, mocker, run_side_effect=ValueError("boom") if failure_stage == "runner" else None) + if failure_stage == "record_load": + generator._records.load.side_effect = ValueError("boom") + hooks = mocker.MagicMock() + mocker.patch("core.ops.base_trace_instance.application_tracing_hooks", return_value=hooks) queue_manager = mocker.MagicMock() self._call(generator, mocker, queue_manager) - assert queue_manager.publish_error.called + hooks.capture_agent_app_terminal.assert_called_once_with( + tenant_id="tenant", + app_id="app1", + message_id="msg", + outcome="failed", + output=None, + ) + queue_manager.publish_error.assert_called_once() diff --git a/api/tests/unit_tests/core/app/apps/agent_chat/test_app_generator.py b/api/tests/unit_tests/core/app/apps/agent_chat/test_app_generator.py index d7308d779451cf..e25488959dd630 100644 --- a/api/tests/unit_tests/core/app/apps/agent_chat/test_app_generator.py +++ b/api/tests/unit_tests/core/app/apps/agent_chat/test_app_generator.py @@ -7,7 +7,7 @@ import pytest from flask import Flask from pydantic import ValidationError -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session from core.app.apps.exc import GenerateTaskStoppedError from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager @@ -73,7 +73,6 @@ def generator( agent_records: tuple[Conversation, Message], monkeypatch: pytest.MonkeyPatch, app_records: AppGenerationRepository, - sqlite_session_factory: sessionmaker[Session], *, annotation_replies: AnnotationReplies, workflow_runtime: WorkflowExecutionDependencies, @@ -85,9 +84,9 @@ def generator( monkeypatch.setattr(TraceQueueManager, "start_timer", lambda _manager: None) return AgentChatAppGenerator( dataset_tools=workflow_runtime.dataset_tools, + tool_invoker=workflow_runtime.agent_tool_invoker, annotations=annotation_replies, records=app_records, - tool_sessions=sqlite_session_factory, workflow_runtime=workflow_runtime, ) diff --git a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py index a4d9f2f81b257f..ef46019cbfb67d 100644 --- a/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py +++ b/api/tests/unit_tests/core/app/apps/test_workflow_app_runner.py @@ -1430,6 +1430,7 @@ def _generate_entity() -> WorkflowAppGenerateEntity: inputs={}, files=[], user_id="user", + user_from=UserFrom.ACCOUNT, stream=True, invoke_from=InvokeFrom.DEBUGGER, workflow_execution_id="run-1", diff --git a/api/tests/unit_tests/core/app/apps/workflow/test_errors.py b/api/tests/unit_tests/core/app/apps/workflow/test_errors.py deleted file mode 100644 index 59b5303255ceaf..00000000000000 --- a/api/tests/unit_tests/core/app/apps/workflow/test_errors.py +++ /dev/null @@ -1,10 +0,0 @@ -from core.app.apps.workflow.errors import WorkflowPausedInBlockingModeError - - -class TestWorkflowErrors: - def test_workflow_paused_in_blocking_mode_error_attributes(self) -> None: - err = WorkflowPausedInBlockingModeError() - assert err.error_code == "workflow_paused_in_blocking_mode" - assert err.code == 400 - assert err.description is not None - assert "blocking response mode" in err.description diff --git a/api/tests/unit_tests/core/app/features/annotation_reply/test_annotation_reply.py b/api/tests/unit_tests/core/app/features/annotation_reply/test_annotation_reply.py deleted file mode 100644 index ccb8c7c5a75ff1..00000000000000 --- a/api/tests/unit_tests/core/app/features/annotation_reply/test_annotation_reply.py +++ /dev/null @@ -1,185 +0,0 @@ -import logging -from types import SimpleNamespace -from unittest.mock import Mock, patch - -import pytest -from sqlalchemy import select -from sqlalchemy.orm import Session - -from core.app.entities.app_invoke_entities import InvokeFrom -from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature -from models.dataset import DatasetCollectionBinding -from models.enums import CollectionBindingType, ConversationFromSource -from models.model import App, AppAnnotationHitHistory, AppAnnotationSetting, Message, MessageAnnotation -from tests.helpers.unit.model_factories import make_app, make_message - -TABLES = (AppAnnotationSetting, DatasetCollectionBinding, MessageAnnotation, AppAnnotationHitHistory) - - -def _app() -> App: - return make_app(icon_type=None, max_active_requests=0) - - -def _message() -> Message: - return make_message(message_id="msg-1", app_id=None, conversation_id=None) - - -def _persist_binding(session: Session) -> DatasetCollectionBinding: - binding = DatasetCollectionBinding( - provider_name="prov", - model_name="model", - type=CollectionBindingType.ANNOTATION, - collection_name="annotation-collection", - ) - session.add(binding) - session.flush() - return binding - - -def _persist_setting( - session: Session, - *, - app_id: str = "app-1", - collection_binding_id: str, - score_threshold: float = 0.5, -) -> AppAnnotationSetting: - setting = AppAnnotationSetting( - app_id=app_id, - score_threshold=score_threshold, - collection_binding_id=collection_binding_id, - created_user_id="user-1", - updated_user_id="user-1", - ) - session.add(setting) - session.flush() - return setting - - -def _persist_annotation(session: Session) -> MessageAnnotation: - annotation = MessageAnnotation( - app_id="app-1", - question="question", - content="content", - account_id="acct-1", - ) - session.add(annotation) - session.flush() - return annotation - - -@pytest.mark.parametrize("sqlite_session", [TABLES], indirect=True) -class TestAnnotationReplyFeature: - def test_query_returns_none_when_setting_missing(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, app_id="other-app", collection_binding_id=binding.id) - - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - - def test_query_returns_none_when_binding_missing(self, sqlite_session: Session): - _persist_setting(sqlite_session, collection_binding_id="missing-binding") - - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - - def test_query_returns_annotation_and_persists_history_for_api(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id, score_threshold=0) - annotation = _persist_annotation(sqlite_session) - document = SimpleNamespace(metadata={"annotation_id": annotation.id, "score": 0.8}) - vector_instance = Mock() - vector_instance.search_by_vector.return_value = [document] - - with patch( - "core.app.features.annotation_reply.annotation_reply.Vector", return_value=vector_instance - ) as vector_cls: - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is annotation - vector_instance.search_by_vector.assert_called_once_with( - query="hi", top_k=1, score_threshold=1, filter={"group_id": ["app-1"]} - ) - assert vector_cls.call_args.kwargs["session"] is sqlite_session - sqlite_session.refresh(annotation) - assert annotation.hit_count == 1 - history = sqlite_session.scalar(select(AppAnnotationHitHistory)) - assert history is not None - assert history.annotation_id == annotation.id - assert history.app_id == "app-1" - assert history.message_id == "msg-1" - assert history.account_id == "user-1" - assert history.source == ConversationFromSource.API - assert history.score == 0.8 - - def test_query_returns_annotation_and_persists_history_for_console(self, sqlite_session: Session): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id) - annotation = _persist_annotation(sqlite_session) - document = SimpleNamespace(metadata={"annotation_id": annotation.id, "score": 0.6}) - vector_instance = Mock() - vector_instance.search_by_vector.return_value = [document] - - with patch("core.app.features.annotation_reply.annotation_reply.Vector", return_value=vector_instance): - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.EXPLORE, - session=sqlite_session, - ) - - assert result is annotation - history = sqlite_session.scalar(select(AppAnnotationHitHistory)) - assert history is not None - assert history.source == ConversationFromSource.CONSOLE - - def test_query_logs_and_returns_none_on_exception(self, sqlite_session: Session, caplog: pytest.LogCaptureFixture): - binding = _persist_binding(sqlite_session) - _persist_setting(sqlite_session, collection_binding_id=binding.id) - vector_instance = Mock() - vector_instance.search_by_vector.side_effect = RuntimeError("boom") - - with ( - patch( - "core.app.features.annotation_reply.annotation_reply.Vector", - return_value=vector_instance, - ), - caplog.at_level(logging.WARNING), - ): - result = AnnotationReplyFeature().query( - app_record=_app(), - message=_message(), - query="hi", - user_id="user-1", - invoke_from=InvokeFrom.SERVICE_API, - session=sqlite_session, - ) - - assert result is None - assert "Query annotation failed" in caplog.text - assert sqlite_session.scalar(select(AppAnnotationHitHistory)) is None - assert sqlite_session.is_active diff --git a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py index 90344e4dc88713..a603f7b501b2c6 100644 --- a/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py +++ b/api/tests/unit_tests/core/app/task_pipeline/test_based_generate_task_pipeline.py @@ -96,6 +96,34 @@ def test_handle_error_preserves_agent_backend_run_failed_error(self, pipeline): assert "Knowledge retrieval failed" in str(err) assert "agent_run_id=run-1" in str(err) + @pytest.mark.parametrize("sqlite_session", [(Message,)], indirect=True) + def test_handle_error_updates_message_when_found(self, pipeline, sqlite_session: Session): + event = QueueErrorEvent(error=ValueError("oops")) + _persist_message(sqlite_session, message_id="msg-1") + + err = pipeline.handle_error(event=event, session=sqlite_session, message_id="msg-1") + + assert err is event.error + sqlite_session.flush() + sqlite_session.expire_all() + updated_message = sqlite_session.get(Message, "msg-1") + assert updated_message is not None + assert updated_message.status == MessageStatus.ERROR + assert updated_message.error == "oops" + + @pytest.mark.parametrize("sqlite_session", [(Message,)], indirect=True) + def test_handle_error_leaves_other_messages_unchanged(self, pipeline, sqlite_session: Session): + event = QueueErrorEvent(error=ValueError("oops")) + _persist_message(sqlite_session, message_id="other-message") + + err = pipeline.handle_error(event=event, session=sqlite_session, message_id="missing-message") + + assert err is event.error + untouched_message = sqlite_session.get(Message, "other-message") + assert untouched_message is not None + assert untouched_message.status == MessageStatus.NORMAL + assert untouched_message.error is None + def test_error_to_stream_response_and_ping(self, pipeline): error_response = pipeline.error_to_stream_response(ValueError("boom")) ping_response = pipeline.ping_stream_response() diff --git a/api/tests/unit_tests/core/mcp/server/test_streamable_http.py b/api/tests/unit_tests/core/mcp/server/test_streamable_http.py index 3667d7a8f5e1e8..27fbcecb2e6f92 100644 --- a/api/tests/unit_tests/core/mcp/server/test_streamable_http.py +++ b/api/tests/unit_tests/core/mcp/server/test_streamable_http.py @@ -50,20 +50,27 @@ def setup_method(self): self.request = types.ClientRequest(root=types.PingRequest()) - def test_handle_ping_request(self): + def test_handle_ping_request(self, *, mock_app_generate: Mock): """Test handling ping request""" # Setup ping request self.request.root = types.PingRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_initialize_request(self): + def test_handle_initialize_request(self, *, mock_app_generate: Mock): """Test handling initialize request""" # Setup initialize request self.request.root = types.InitializeRequest( @@ -75,32 +82,54 @@ def test_handle_initialize_request(self): ) result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_list_tools_request(self): + def test_handle_list_tools_request(self, *, mock_app_generate: Mock): """Test handling list tools request""" # Setup list tools request self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.jsonrpc == "2.0" assert result.id == 123 - def test_handle_list_tools_request_threads_protocol_version(self): + def test_handle_list_tools_request_threads_protocol_version(self, *, mock_app_generate: Mock): """The negotiated version reaches handle_list_tools through the dispatcher.""" self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2025-06-18" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2025-06-18", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) @@ -108,19 +137,26 @@ def test_handle_list_tools_request_threads_protocol_version(self): assert tool["outputSchema"] == {"type": "object"} assert tool["title"] == "test_app" - def test_handle_list_tools_request_legacy_serialization_unchanged(self): + def test_handle_list_tools_request_legacy_serialization_unchanged(self, *, mock_app_generate: Mock): """A 2024-11-05 tools/list response serializes without any 2025-06-18 fields.""" self.request.root = types.ListToolsRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2024-11-05" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2024-11-05", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) tool = result.result["tools"][0] assert set(tool) == {"name", "description", "inputSchema"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request(self, mock_app_generate): """Test handling call tool request""" # Setup call tool request @@ -135,7 +171,14 @@ def test_handle_call_tool_request(self, mock_app_generate): mock_app_generate.generate.return_value = mock_response result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) @@ -145,7 +188,6 @@ def test_handle_call_tool_request(self, mock_app_generate): # Verify AppGenerateService was called mock_app_generate.generate.assert_called_once() - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_returns_trigger_workflow_business_error(self, mock_app_generate): call_request = types.CallToolRequest( params=types.CallToolRequestParams(name="test_app", arguments={"query": "test question"}) @@ -161,13 +203,13 @@ def test_handle_call_tool_returns_trigger_workflow_business_error(self, mock_app self.mcp_server, self.end_user, 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) assert result.error.code == types.INVALID_REQUEST assert result.error.data == {"code": "trigger_workflow_service_mode_unavailable"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request_threads_protocol_version(self, mock_app_generate): """The negotiated version reaches handle_call_tool through the dispatcher.""" call_request = types.CallToolRequest( @@ -178,13 +220,20 @@ def test_handle_call_tool_request_threads_protocol_version(self, mock_app_genera mock_app_generate.generate.return_value = {"answer": "test answer"} result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2025-06-18" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2025-06-18", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert result.result["structuredContent"] == {"answer": "test answer"} - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_request_legacy_serialization_unchanged(self, mock_app_generate): """A 2024-11-05 tools/call response serializes without structuredContent.""" call_request = types.CallToolRequest( @@ -195,21 +244,36 @@ def test_handle_call_tool_request_legacy_serialization_unchanged(self, mock_app_ mock_app_generate.generate.return_value = {"answer": "test answer"} result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123, "2024-11-05" + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + "2024-11-05", + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCResponse) assert "structuredContent" not in result.result assert result.result["content"][0]["text"] == "test answer" - def test_handle_unknown_request_type(self): + def test_handle_unknown_request_type(self, *, mock_app_generate: Mock): """Test handling unknown request type""" # A valid request type with no application-side handler. self.request.root = types.ListResourcesRequest() result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) @@ -217,18 +281,27 @@ def test_handle_unknown_request_type(self): assert result.id == 123 assert result.error.code == types.METHOD_NOT_FOUND - def test_handle_value_error(self): + def test_handle_value_error(self, *, mock_app_generate: Mock): """Test handling ValueError""" # Setup request that will cause ValueError self.request.root = types.CallToolRequest(params=types.CallToolRequestParams(name="test_app", arguments={})) # Don't provide end_user to cause ValueError - result = handle_mcp_request(Mock(), self.app, self.request, self.user_input_form, self.mcp_server, None, 123) + result = handle_mcp_request( + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + None, + 123, + generate=mock_app_generate.generate, + ) assert isinstance(result, types.JSONRPCError) assert result.error.code == types.INVALID_PARAMS - def test_handle_generic_exception(self): + def test_handle_generic_exception(self, *, mock_app_generate: Mock): """Test handling generic exception""" # Setup request that will cause generic exception self.request.root = types.PingRequest() @@ -236,7 +309,14 @@ def test_handle_generic_exception(self): # Inject the handler failure while preserving normal request dispatch. with patch("core.mcp.server.streamable_http.handle_ping", side_effect=Exception("Test error")): result = handle_mcp_request( - Mock(), self.app, self.request, self.user_input_form, self.mcp_server, self.end_user, 123 + Mock(), + self.app, + self.request, + self.user_input_form, + self.mcp_server, + self.end_user, + 123, + generate=mock_app_generate.generate, ) assert isinstance(result, types.JSONRPCError) @@ -323,7 +403,6 @@ def test_handle_list_tools_omits_structured_output_for_intermediate_client(self) assert tool.outputSchema is None assert tool.title is None - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool(self, mock_app_generate): """Test call tool handler""" app = App( @@ -342,7 +421,7 @@ def test_handle_call_tool(self, mock_app_generate): mock_response = {"answer": "test answer"} mock_app_generate.generate.return_value = mock_response - result = handle_call_tool(Mock(), app, request, user_input_form, end_user) + result = handle_call_tool(Mock(), app, request, user_input_form, end_user, generate=mock_app_generate.generate) assert isinstance(result, types.CallToolResult) assert len(result.content) == 1 @@ -351,7 +430,6 @@ def test_handle_call_tool(self, mock_app_generate): assert hasattr(text_content, "text") assert text_content.text == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_structured_output_modern_client(self, mock_app_generate): """structuredContent is attached alongside TextContent for >= 2025-06-18.""" app = App( @@ -365,12 +443,13 @@ def test_handle_call_tool_structured_output_modern_client(self, mock_app_generat mock_app_generate.generate.return_value = {"answer": "test answer"} - result = handle_call_tool(Mock(), app, request, [], EndUser(), "2025-06-18") + result = handle_call_tool( + Mock(), app, request, [], EndUser(), "2025-06-18", generate=mock_app_generate.generate + ) assert result.structuredContent == {"answer": "test answer"} assert result.content[0].text == "test answer" - @patch("core.mcp.server.streamable_http.AppGenerateService") def test_handle_call_tool_no_structured_output_legacy_client(self, mock_app_generate): """structuredContent is omitted for 2024-11-05 clients.""" app = App( @@ -384,19 +463,21 @@ def test_handle_call_tool_no_structured_output_legacy_client(self, mock_app_gene mock_app_generate.generate.return_value = {"answer": "test answer"} - result = handle_call_tool(Mock(), app, request, [], EndUser(), "2024-11-05") + result = handle_call_tool( + Mock(), app, request, [], EndUser(), "2024-11-05", generate=mock_app_generate.generate + ) assert result.structuredContent is None assert result.content[0].text == "test answer" - def test_handle_call_tool_no_end_user(self): + def test_handle_call_tool_no_end_user(self, *, mock_app_generate: Mock): """Test call tool handler without end user""" app = App() request = types.ClientRequest(root=types.CallToolRequest(params=types.CallToolRequestParams(name="test_app"))) user_input_form: list[VariableEntity] = [] with pytest.raises(ValueError, match="End user not found"): - handle_call_tool(Mock(), app, request, user_input_form, None) + handle_call_tool(Mock(), app, request, user_input_form, None, generate=mock_app_generate.generate) class TestUtilityFunctions: @@ -833,3 +914,8 @@ def test_supported_header_passes_through(self): def test_unsupported_header_returns_none(self): """An explicit but unsupported header signals an error (None).""" assert negotiate_protocol_version("1999-01-01", False) is None + + +@pytest.fixture +def mock_app_generate() -> Mock: + return Mock() diff --git a/api/tests/unit_tests/core/plugin/backwards_invocation/test_app.py b/api/tests/unit_tests/core/plugin/backwards_invocation/test_app.py index 328b025690efb0..0c7871d4b0ea0f 100644 --- a/api/tests/unit_tests/core/plugin/backwards_invocation/test_app.py +++ b/api/tests/unit_tests/core/plugin/backwards_invocation/test_app.py @@ -9,13 +9,14 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.layers.pause_state_persist_layer import PauseStateLayerConfig -from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation from core.plugin.backwards_invocation.base import BaseBackwardsInvocation from models import Account, Tenant, TenantAccountJoin from models.enums import EndUserType from models.model import App, AppMode, AppModelConfig, EndUser from models.workflow import Workflow -from services.workflow_run_agg import WorkflowRunAgg +from repositories.workflow.definition_repository import workflow_snapshot +from services.app.generation.runtime import AppGenerationRuntime +from services.plugin.app_invocation import PluginAppBackwardsInvocation from tests.helpers.unit.model_factories import make_app, make_end_user, make_workflow @@ -124,7 +125,7 @@ def _real_sessions( self.session_factory = sqlite_session_factory self.sqlite_engine = sqlite_engine self.end_users = _EndUserProvisioner() - mocker.patch("core.plugin.backwards_invocation.app.create_session", side_effect=sqlite_session_factory) + mocker.patch("services.plugin.app_invocation.create_session", side_effect=sqlite_session_factory) def test_fetch_app_info_workflow_path(self, mocker: MockerFixture): variable = {"type": "text-input", "variable": "foo", "label": "Foo", "required": False} @@ -135,7 +136,7 @@ def test_fetch_app_info_workflow_path(self, mocker: MockerFixture): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=workflow) mapper = mocker.patch( - "core.plugin.backwards_invocation.app.get_parameters_from_feature_dict", + "services.plugin.app_invocation.get_parameters_from_feature_dict", return_value={"mapped": True}, ) @@ -150,7 +151,7 @@ def test_fetch_app_info_model_config_path(self, mocker: MockerFixture): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_app_model_config_dict", return_value=model_config_dict) mocker.patch( - "core.plugin.backwards_invocation.app.get_parameters_from_feature_dict", + "services.plugin.app_invocation.get_parameters_from_feature_dict", return_value={"mapped": True}, ) @@ -168,7 +169,9 @@ def test_fetch_app_info_model_config_path(self, mocker: MockerFixture): (AppMode.COMPLETION, "invoke_completion_app"), ], ) - def test_invoke_app_routes_by_mode(self, mocker: MockerFixture, mode, route_method): + def test_invoke_app_routes_by_mode( + self, mocker: MockerFixture, mode, route_method, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=mode) user = _end_user() workflow = _workflow() @@ -178,7 +181,6 @@ def test_invoke_app_routes_by_mode(self, mocker: MockerFixture, mode, route_meth route = mocker.patch.object(PluginAppBackwardsInvocation, route_method, return_value={"routed": True}) result = PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="user", tenant_id="tenant", @@ -189,12 +191,15 @@ def test_invoke_app_routes_by_mode(self, mocker: MockerFixture, mode, route_meth files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"routed": True} assert route.call_count == 1 - def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixture): + def test_invoke_app_uses_end_user_when_user_id_missing( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user() workflow = _workflow() @@ -204,7 +209,6 @@ def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixtu route = mocker.patch.object(PluginAppBackwardsInvocation, "invoke_workflow_app", return_value={"ok": True}) result = PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="", tenant_id="tenant", @@ -215,6 +219,7 @@ def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixtu files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"ok": True} @@ -222,13 +227,14 @@ def test_invoke_app_uses_end_user_when_user_id_missing(self, mocker: MockerFixtu assert route.call_args.args[1] is workflow assert route.call_args.args[2] is end_user - def test_invoke_app_missing_query_for_chat_raises(self, mocker: MockerFixture): + def test_invoke_app_missing_query_for_chat_raises( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=_app(mode=AppMode.CHAT)) mocker.patch.object(PluginAppBackwardsInvocation, "_get_user", return_value=_end_user()) with pytest.raises(ValueError, match="missing query"): PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="user", tenant_id="tenant", @@ -239,9 +245,10 @@ def test_invoke_app_missing_query_for_chat_raises(self, mocker: MockerFixture): files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) - def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture): + def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime): mocker.patch.object( PluginAppBackwardsInvocation, "_get_app", @@ -251,7 +258,6 @@ def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture): with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="user", tenant_id="tenant", @@ -262,21 +268,29 @@ def test_invoke_app_unexpected_mode_raises(self, mocker: MockerFixture): files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) @pytest.mark.parametrize( ("mode", "generator_path"), [ - (AppMode.AGENT_CHAT, "core.plugin.backwards_invocation.app.AgentChatAppGenerator.generate"), - (AppMode.CHAT, "core.plugin.backwards_invocation.app.ChatAppGenerator.generate"), + (AppMode.AGENT_CHAT, "services.plugin.app_invocation.AgentChatAppGenerator.generate"), + (AppMode.CHAT, "services.plugin.app_invocation.ChatAppGenerator.generate"), ], ) - def test_invoke_chat_app_agent_and_chat(self, mocker: MockerFixture, mode, generator_path): + def test_invoke_chat_app_agent_and_chat( + self, mocker: MockerFixture, mode, generator_path, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=mode) - spy = mocker.patch(generator_path, return_value={"result": "ok"}) + agent_generator = None + if mode is AppMode.AGENT_CHAT: + agent_generator = mocker.patch("services.plugin.app_invocation.AgentChatAppGenerator") + spy = agent_generator.return_value.generate + spy.return_value = {"result": "ok"} + else: + spy = mocker.patch(generator_path, return_value={"result": "ok"}) result = PluginAppBackwardsInvocation.invoke_chat_app( - execution_driver=WorkflowRunAgg.run, app=app, user=_end_user(), conversation_id="conv-1", @@ -285,12 +299,17 @@ def test_invoke_chat_app_agent_and_chat(self, mocker: MockerFixture, mode, gener inputs={"k": "v"}, files=[], session=self.session, + runtime=workflow_runtime, ) assert result == {"result": "ok"} assert spy.call_count == 1 + if agent_generator is not None: + assert agent_generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker - def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: MockerFixture): + def test_invoke_chat_app_advanced_chat_injects_pause_state_config( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): workflow = _workflow() workflow.created_by = "owner-id" @@ -298,17 +317,15 @@ def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=workflow) mocker.patch( - "core.plugin.backwards_invocation.app.db", + "services.plugin.app_invocation.db", _DatabaseWithEngine(self.sqlite_engine), ) - generator_spy = mocker.patch( - "core.plugin.backwards_invocation.app.AdvancedChatAppGenerator.generate", - return_value={"result": "ok"}, - ) + generator_factory = mocker.patch("services.plugin.app_invocation.AdvancedChatAppGenerator") + generator_spy = generator_factory.return_value.generate + generator_spy.return_value = {"result": "ok"} session = self.session result = PluginAppBackwardsInvocation.invoke_chat_app( - execution_driver=WorkflowRunAgg.run, app=app, user=_end_user(), conversation_id="conv-1", @@ -317,21 +334,25 @@ def test_invoke_chat_app_advanced_chat_injects_pause_state_config(self, mocker: inputs={"k": "v"}, files=[], session=session, + runtime=workflow_runtime, ) assert result == {"result": "ok"} + assert generator_factory.call_args.kwargs["runtime"] is workflow_runtime + assert generator_factory.call_args.kwargs["execution_driver"] is workflow_runtime.execution_driver call_kwargs = generator_spy.call_args.kwargs - assert call_kwargs["session"] is session + assert "session" not in call_kwargs pause_state_config = call_kwargs.get("pause_state_config") assert isinstance(pause_state_config, PauseStateLayerConfig) assert pause_state_config.state_owner_user_id == "owner-id" - def test_invoke_chat_app_advanced_chat_without_workflow_raises(self, mocker: MockerFixture): + def test_invoke_chat_app_advanced_chat_without_workflow_raises( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=AppMode.ADVANCED_CHAT) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None) with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_chat_app( - execution_driver=WorkflowRunAgg.run, app=app, user=_end_user(), conversation_id="conv-1", @@ -340,13 +361,13 @@ def test_invoke_chat_app_advanced_chat_without_workflow_raises(self, mocker: Moc inputs={}, files=[], session=self.session, + runtime=workflow_runtime, ) - def test_invoke_chat_app_unexpected_mode_raises(self): + def test_invoke_chat_app_unexpected_mode_raises(self, *, workflow_runtime: AppGenerationRuntime): app = _app(mode=cast(AppMode, "invalid")) with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_chat_app( - execution_driver=WorkflowRunAgg.run, app=app, user=_end_user(), conversation_id="conv-1", @@ -355,47 +376,52 @@ def test_invoke_chat_app_unexpected_mode_raises(self): inputs={}, files=[], session=self.session, + runtime=workflow_runtime, ) - def test_invoke_workflow_app_injects_pause_state_config(self, mocker: MockerFixture): + def test_invoke_workflow_app_injects_pause_state_config( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): workflow = _workflow() workflow.created_by = "owner-id" app = _app(mode=AppMode.WORKFLOW) mocker.patch( - "core.plugin.backwards_invocation.app.db", + "services.plugin.app_invocation.db", _DatabaseWithEngine(self.sqlite_engine), ) - generator_spy = mocker.patch( - "core.plugin.backwards_invocation.app.WorkflowAppGenerator.generate", - return_value={"result": "ok"}, - ) + generator_factory = mocker.patch("services.plugin.app_invocation.WorkflowAppGenerator") + generator_spy = generator_factory.return_value.generate + generator_spy.return_value = {"result": "ok"} result = PluginAppBackwardsInvocation.invoke_workflow_app( - execution_driver=WorkflowRunAgg.run, app=app, - workflow=workflow, + workflow=workflow_snapshot(workflow), user=_end_user(), stream=False, inputs={"k": "v"}, files=[], + runtime=workflow_runtime, ) assert result == {"result": "ok"} + assert generator_factory.call_args.kwargs["runtime"] is workflow_runtime + assert generator_factory.call_args.kwargs["execution_driver"] is workflow_runtime.execution_driver call_kwargs = generator_spy.call_args.kwargs pause_state_config = call_kwargs.get("pause_state_config") assert isinstance(pause_state_config, PauseStateLayerConfig) assert pause_state_config.state_owner_user_id == "owner-id" - def test_invoke_app_workflow_without_workflow_raises(self, mocker: MockerFixture): + def test_invoke_app_workflow_without_workflow_raises( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=AppMode.WORKFLOW) mocker.patch.object(PluginAppBackwardsInvocation, "_get_app", return_value=app) mocker.patch.object(PluginAppBackwardsInvocation, "_get_user", return_value=_end_user()) mocker.patch.object(PluginAppBackwardsInvocation, "_get_workflow", return_value=None) with pytest.raises(ValueError, match="unexpected app type"): PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="user", tenant_id="tenant", @@ -406,15 +432,24 @@ def test_invoke_app_workflow_without_workflow_raises(self, mocker: MockerFixture files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) - def test_invoke_completion_app(self, mocker: MockerFixture): - spy = mocker.patch( - "core.plugin.backwards_invocation.app.CompletionAppGenerator.generate", return_value={"ok": 1} - ) + def test_invoke_completion_app(self, workflow_runtime, mocker: MockerFixture, app_records, annotation_replies): + spy = mocker.patch("services.plugin.app_invocation.CompletionAppGenerator.generate", return_value={"ok": 1}) app = _app(mode=AppMode.COMPLETION) - result = PluginAppBackwardsInvocation.invoke_completion_app(app, _end_user(), False, {"x": 1}, [], self.session) + result = PluginAppBackwardsInvocation.invoke_completion_app( + app, + _end_user(), + False, + {"x": 1}, + [], + self.session, + records=app_records, + annotations=annotation_replies, + retrieval=workflow_runtime.retrieval, + ) assert result == {"ok": 1} assert spy.call_count == 1 @@ -535,7 +570,9 @@ def test_get_user_raises_when_user_not_found(self): with pytest.raises(ValueError, match="user not found"): PluginAppBackwardsInvocation._get_user("uid", app) - def test_invoke_app_creates_end_user_for_unknown_external_user_id(self, mocker: MockerFixture): + def test_invoke_app_creates_end_user_for_unknown_external_user_id( + self, mocker: MockerFixture, *, workflow_runtime: AppGenerationRuntime + ): app = _app(mode=AppMode.WORKFLOW) end_user = _end_user(session_id="wecom-sender-1") workflow = _workflow() @@ -546,7 +583,6 @@ def test_invoke_app_creates_end_user_for_unknown_external_user_id(self, mocker: route = mocker.patch.object(PluginAppBackwardsInvocation, "invoke_workflow_app", return_value={"ok": True}) result = PluginAppBackwardsInvocation.invoke_app( - execution_driver=WorkflowRunAgg.run, app_id="app", user_id="wecom-sender-1", tenant_id="tenant", @@ -557,6 +593,7 @@ def test_invoke_app_creates_end_user_for_unknown_external_user_id(self, mocker: files=[], session=self.session, end_users=self.end_users, + runtime=workflow_runtime, ) assert result == {"ok": True} @@ -623,7 +660,7 @@ def test_get_app_model_config_dict_uses_explicit_session_for_annotation_reply(se self.session.add(app_model_config) self.session.commit() load_annotation_reply_config = mocker.patch( - "core.plugin.backwards_invocation.app.load_annotation_reply_config", + "services.plugin.app_invocation.load_annotation_reply_config", return_value=annotation_reply, ) app = _app(app_model_config_id="config-1") diff --git a/api/tests/unit_tests/core/tools/__base/test_tool.py b/api/tests/unit_tests/core/tools/__base/test_tool.py index d32f62bae82769..c81daca25f8240 100644 --- a/api/tests/unit_tests/core/tools/__base/test_tool.py +++ b/api/tests/unit_tests/core/tools/__base/test_tool.py @@ -52,7 +52,7 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, - session: Any, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -94,7 +94,7 @@ def _build_tool(runtime: ToolRuntime | None = None) -> DummyTool: return DummyTool(entity=entity, runtime=runtime) -def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Session): +def test_invoke_supports_single_message_and_parameter_casting(unbound_session: Session): runtime = ToolRuntime( tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER, @@ -112,7 +112,7 @@ def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Se messages = list( tool.invoke( - session=sqlite_session, + session=unbound_session, user_id="user-1", tool_parameters={"age": "18", "raw": "keep"}, conversation_id="conv-1", @@ -132,7 +132,7 @@ def test_invoke_supports_single_message_and_parameter_casting(sqlite_session: Se } -def test_invoke_preserves_multiple_select_values(sqlite_session: Session): +def test_invoke_preserves_multiple_select_values(unbound_session: Session): tool = _build_tool() parameter = ToolParameter.get_simple_instance( name="choice", @@ -144,22 +144,22 @@ def test_invoke_preserves_multiple_select_values(sqlite_session: Session): parameter.multiple = True tool.entity.parameters = [parameter] - list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"choice": ["a", "b"]})) + list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"choice": ["a", "b"]})) assert tool.last_invocation is not None assert tool.last_invocation["tool_parameters"] == {"choice": ["a", "b"]} # A legacy comma-separated string is normalized, not rejected (#45003). - list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"choice": "a,b"})) + list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"choice": "a,b"})) assert tool.last_invocation["tool_parameters"] == {"choice": ["a", "b"]} # A non-list, non-string value is still rejected. with pytest.raises(ValueError, match="must be a list"): - tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"choice": 1}) + tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"choice": 1}) -def test_invoke_supports_list_and_generator_results(sqlite_session: Session): +def test_invoke_supports_list_and_generator_results(unbound_session: Session): tool = _build_tool() tool.result = [tool.create_text_message("a"), tool.create_text_message("b")] - list_messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={})) + list_messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={})) assert [msg.message.text for msg in list_messages] == ["a", "b"] def _message_generator() -> Generator[ToolInvokeMessage]: @@ -167,7 +167,7 @@ def _message_generator() -> Generator[ToolInvokeMessage]: yield tool.create_text_message("g2") tool.result = _message_generator() - generated_messages = list(tool.invoke(session=sqlite_session, user_id="user-2", tool_parameters={})) + generated_messages = list(tool.invoke(session=unbound_session, user_id="user-2", tool_parameters={})) assert [msg.message.text for msg in generated_messages] == ["g1", "g2"] @@ -376,6 +376,6 @@ def test_message_factory_helpers(): assert variable_message.message.stream is False -def test_base_abstract_invoke_placeholder_returns_none(sqlite_session: Session): +def test_base_abstract_invoke_placeholder_returns_none(unbound_session: Session): tool = _build_tool() - assert Tool._invoke(tool, session=sqlite_session, user_id="u", tool_parameters={}) is None + assert Tool._invoke(tool, session=unbound_session, user_id="u", tool_parameters={}) is None diff --git a/api/tests/unit_tests/core/tools/builtin_tool/providers/time/tools/test_localtime_to_timestamp.py b/api/tests/unit_tests/core/tools/builtin_tool/providers/time/tools/test_localtime_to_timestamp.py index 09da0178560201..2a477ae75482db 100644 --- a/api/tests/unit_tests/core/tools/builtin_tool/providers/time/tools/test_localtime_to_timestamp.py +++ b/api/tests/unit_tests/core/tools/builtin_tool/providers/time/tools/test_localtime_to_timestamp.py @@ -51,24 +51,24 @@ def _raise_runtime_error(*_args: object, **_kwargs: object) -> None: raise RuntimeError("boom") -def test_current_time_tool(sqlite_session: Session): +def test_current_time_tool(unbound_session: Session): current_tool = _build_builtin_tool(CurrentTimeTool) - utc_text = list(current_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"timezone": "UTC"}))[ + utc_text = list(current_tool.invoke(session=unbound_session, user_id="u", tool_parameters={"timezone": "UTC"}))[ 0 ].message.text assert utc_text invalid_tz = list( - current_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"timezone": "Invalid/TZ"}) + current_tool.invoke(session=unbound_session, user_id="u", tool_parameters={"timezone": "Invalid/TZ"}) )[0].message.text assert "Invalid timezone" in invalid_tz -def test_localtime_to_timestamp_tool(sqlite_session: Session): +def test_localtime_to_timestamp_tool(unbound_session: Session): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) ts_message = list( localtime_tool.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={"localtime": "2024-01-01 10:00:00", "timezone": "UTC"}, ) @@ -94,11 +94,13 @@ def test_localtime_to_timestamp_tool(sqlite_session: Session): LocaltimeToTimestampTool.localtime_to_timestamp("bad", "%Y-%m-%d %H:%M:%S", "UTC") -def test_timestamp_to_localtime_tool(sqlite_session: Session): +def test_timestamp_to_localtime_tool(unbound_session: Session): to_local_tool = _build_builtin_tool(TimestampToLocaltimeTool) local_text = list( to_local_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"timestamp": 1704067200, "timezone": "UTC"} + session=unbound_session, + user_id="u", + tool_parameters={"timestamp": 1704067200, "timezone": "UTC"}, ) )[0].message.text assert "2024" in local_text @@ -106,11 +108,11 @@ def test_timestamp_to_localtime_tool(sqlite_session: Session): TimestampToLocaltimeTool.timestamp_to_localtime("bad", "UTC") # type: ignore[arg-type] -def test_timezone_conversion_tool(sqlite_session: Session): +def test_timezone_conversion_tool(unbound_session: Session): timezone_tool = _build_builtin_tool(TimezoneConversionTool) converted = list( timezone_tool.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={ "current_time": "2024-01-01 08:00:00", @@ -124,10 +126,14 @@ def test_timezone_conversion_tool(sqlite_session: Session): TimezoneConversionTool.timezone_convert("bad", "UTC", "Asia/Tokyo") -def test_weekday_tool(sqlite_session: Session): +def test_weekday_tool(unbound_session: Session): weekday_tool = _build_builtin_tool(WeekdayTool) valid = list( - weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "month": 1, "day": 1}) + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "month": 1, "day": 1}, + ) )[0].message.text expected_date = date(2024, 1, 1) expected_message = ( @@ -137,16 +143,28 @@ def test_weekday_tool(sqlite_session: Session): ) assert valid == expected_message invalid = list( - weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "month": 2, "day": 31}) + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "month": 2, "day": 31}, + ) )[0].message.text assert "Invalid date" in invalid with pytest.raises(ValueError, match="Month is required"): - list(weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"year": 2024, "day": 1})) + list( + weekday_tool.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"year": 2024, "day": 1}, + ) + ) # LLMs often send numeric parameters as strings; these must be accepted. string_params = list( weekday_tool.invoke( - session=sqlite_session, user_id="u", tool_parameters={"year": "2024", "month": "3", "day": "5"} + session=unbound_session, + user_id="u", + tool_parameters={"year": "2024", "month": "3", "day": "5"}, ) )[0].message.text expected_date = date(2024, 3, 5) @@ -157,11 +175,11 @@ def test_weekday_tool(sqlite_session: Session): ) # Missing or non-numeric values must yield an "Invalid date" message, not crash. for params in ({"year": 2024, "month": 3}, {"year": "abc", "month": 3, "day": 5}): - result = list(weekday_tool.invoke(session=sqlite_session, user_id="u", tool_parameters=params))[0].message.text + result = list(weekday_tool.invoke(session=unbound_session, user_id="u", tool_parameters=params))[0].message.text assert "Invalid date" in result -def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -170,7 +188,7 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_ses ) result = list( simple_code.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) @@ -178,18 +196,20 @@ def test_simple_code_valid_execution(monkeypatch: pytest.MonkeyPatch, sqlite_ses assert result == "ok" -def test_simple_code_invalid_language(sqlite_session: Session): +def test_simple_code_invalid_language(unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) with pytest.raises(ValueError, match="Only python3 and javascript"): list( simple_code.invoke( - session=sqlite_session, user_id="u", tool_parameters={"language": "go", "code": "fmt.Println(1)"} + session=unbound_session, + user_id="u", + tool_parameters={"language": "go", "code": "fmt.Println(1)"}, ) ) -def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): simple_code = _build_builtin_tool(SimpleCode) monkeypatch.setattr( @@ -199,35 +219,39 @@ def test_simple_code_execution_error(monkeypatch: pytest.MonkeyPatch, sqlite_ses with pytest.raises(ToolInvokeError, match="boom"): list( simple_code.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={"language": "python3", "code": "print(1)"}, ) ) -def test_webscraper_empty_url(sqlite_session: Session): +def test_webscraper_empty_url(unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) - empty = list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": ""}))[0].message.text + empty = list(webscraper.invoke(session=unbound_session, user_id="u", tool_parameters={"url": ""}))[0].message.text assert empty == "Please input url" -def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_fetch(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") - full = list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": "https://example.com"}))[ - 0 - ].message.text + full = list( + webscraper.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"url": "https://example.com"}, + ) + )[0].message.text assert full == "page" -def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr("core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", lambda *a, **k: "page") monkeypatch.setattr(webscraper, "summary", lambda user_id, content: "summary") summarized = list( webscraper.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={"url": "https://example.com", "generate_summary": True}, ) @@ -235,26 +259,32 @@ def test_webscraper_summary(monkeypatch: pytest.MonkeyPatch, sqlite_session: Ses assert summarized == "summary" -def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_webscraper_fetch_error(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): webscraper = _build_builtin_tool(WebscraperTool) monkeypatch.setattr( "core.tools.builtin_tool.providers.webscraper.tools.webscraper.get_url", _raise_runtime_error, ) with pytest.raises(ToolInvokeError, match="boom"): - list(webscraper.invoke(session=sqlite_session, user_id="u", tool_parameters={"url": "https://example.com"})) + list( + webscraper.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"url": "https://example.com"}, + ) + ) -def test_asr_invalid_file(sqlite_session: Session): +def test_asr_invalid_file(unbound_session: Session): asr = _build_builtin_tool(ASRTool) file_obj = SimpleNamespace(type=FileType.DOCUMENT) - invalid_file = list(asr.invoke(session=sqlite_session, user_id="u", tool_parameters={"audio_file": file_obj}))[ + invalid_file = list(asr.invoke(session=unbound_session, user_id="u", tool_parameters={"audio_file": file_obj}))[ 0 ].message.text assert "not a valid audio file" in invalid_file -def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): asr = _build_builtin_tool(ASRTool) model_instance = type("M", (), {"invoke_speech2text": lambda self, file: "transcript"})() model_manager = type("Mgr", (), {"get_model_instance": lambda *a, **k: model_instance})() @@ -267,7 +297,11 @@ def test_asr_valid_file_invocation(monkeypatch: pytest.MonkeyPatch, sqlite_sessi ) audio_file = SimpleNamespace(type=FileType.AUDIO) ok = list( - asr.invoke(session=sqlite_session, user_id="u", tool_parameters={"audio_file": audio_file, "model": "p#m"}) + asr.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"audio_file": audio_file, "model": "p#m"}, + ) )[0].message.text assert ok == "transcript" assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -284,7 +318,7 @@ def test_asr_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyP assert asr.get_runtime_parameters()[0].name == "model" -def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, unbound_session: Session): tts = _build_builtin_tool(TTSTool) captured_manager_kwargs = {} voices_model_instance = type( @@ -302,7 +336,13 @@ def test_tts_invoke_returns_messages(monkeypatch: pytest.MonkeyPatch, sqlite_ses or type("M", (), {"get_model_instance": lambda *a, **k: voices_model_instance})() ), ) - messages = list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + messages = list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.TEXT, ToolInvokeMessage.MessageType.BLOB] assert messages[1].meta == {"mime_type": "audio/mpeg"} assert captured_manager_kwargs == {"tenant_id": "tenant-1", "user_id": "u"} @@ -315,18 +355,24 @@ def test_tts_get_available_models_requires_runtime(): tts.get_available_models() -def test_tts_tool_raises_when_runtime_missing(sqlite_session: Session): +def test_tts_tool_raises_when_runtime_missing(unbound_session: Session): tts = _build_builtin_tool(TTSTool) tts.runtime = None with pytest.raises(ValueError, match="Runtime is required"): - list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) @pytest.mark.parametrize( "voices", [[{"value": None}], []], ) -def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, sqlite_session: Session): +def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, unbound_session: Session): tts = _build_builtin_tool(TTSTool) tts.runtime = ToolRuntime(tenant_id="tenant-1", invoke_from=InvokeFrom.DEBUGGER) model_without_voice = type( @@ -342,7 +388,13 @@ def test_tts_tool_raises_when_voice_unavailable(monkeypatch, voices, sqlite_sess lambda **_: type("Manager", (), {"get_model_instance": lambda *args, **kwargs: model_without_voice})(), ) with pytest.raises(ValueError, match="no voice available"): - list(tts.invoke(session=sqlite_session, user_id="u", tool_parameters={"model": "p#m", "text": "hello"})) + list( + tts.invoke( + session=unbound_session, + user_id="u", + tool_parameters={"model": "p#m", "text": "hello"}, + ) + ) def test_tts_tool_get_available_models_and_runtime_parameters(monkeypatch: pytest.MonkeyPatch): @@ -394,14 +446,14 @@ def test_provider_classes_and_builtin_sort(monkeypatch: pytest.MonkeyPatch): assert [p.name for p in sorted_providers] == ["a", "b"] -def test_localtime_to_timestamp_tool_epoch_zero(sqlite_session: Session): +def test_localtime_to_timestamp_tool_epoch_zero(unbound_session: Session): localtime_tool = _build_builtin_tool(LocaltimeToTimestampTool) # 1970-01-01 08:00:00 in Asia/Shanghai is exactly Unix epoch 0. A valid # conversion to timestamp 0 must not be reported as "Invalid localtime" # just because 0 is falsy. epoch_message = list( localtime_tool.invoke( - session=sqlite_session, + session=unbound_session, user_id="u", tool_parameters={"localtime": "1970-01-01 08:00:00", "timezone": "Asia/Shanghai"}, ) diff --git a/api/tests/unit_tests/core/tools/custom_tool/test_tool.py b/api/tests/unit_tests/core/tools/custom_tool/test_tool.py index ae84950513822d..d4672ebaf57eed 100644 --- a/api/tests/unit_tests/core/tools/custom_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/custom_tool/test_tool.py @@ -309,7 +309,7 @@ def _fake_get(url, **kwargs): @staticmethod def test_do_http_request_handles_file_upload_and_invoke_paths( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, unbound_session: Session ): openapi = { "parameters": [], @@ -350,12 +350,12 @@ def _fake_post(url, **kwargs): monkeypatch.setattr(tool, "assembling_request", lambda parameters: {}) monkeypatch.setattr(tool, "do_http_request", lambda *args, **kwargs: httpx.Response(200, text='{"a":1}')) monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse({"a": 1}, True)) - messages = list(tool.invoke(session=sqlite_session, user_id="u1", tool_parameters={})) + messages = list(tool.invoke(session=unbound_session, user_id="u1", tool_parameters={})) assert [m.type for m in messages] == [ToolInvokeMessage.MessageType.JSON, ToolInvokeMessage.MessageType.TEXT] # _invoke text path monkeypatch.setattr(tool, "validate_and_parse_response", lambda _: ParsedResponse("plain", False)) - messages = list(tool.invoke(session=sqlite_session, user_id="u1", tool_parameters={})) + messages = list(tool.invoke(session=unbound_session, user_id="u1", tool_parameters={})) assert len(messages) == 1 assert messages[0].message.text == "plain" diff --git a/api/tests/unit_tests/core/tools/mcp_tool/test_tool.py b/api/tests/unit_tests/core/tools/mcp_tool/test_tool.py index 6943ab8d7df0bb..64d68758e433d1 100644 --- a/api/tests/unit_tests/core/tools/mcp_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/mcp_tool/test_tool.py @@ -145,8 +145,7 @@ def test_mcp_tool_usage_extraction_helpers(): assert derived.total_tokens == 0 @staticmethod - @pytest.mark.parametrize("sqlite_session", [()], indirect=True) - def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_session: Session): + def test_mcp_tool_invoke_handles_content_types_and_structured_output(unbound_session: Session): tool = _build_mcp_tool() img_data = base64.b64encode(b"img").decode() blob_data = base64.b64encode(b"blob").decode() @@ -172,7 +171,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_sess ) with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): - messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"a": 1})) + messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"a": 1})) types = [m.type for m in messages] assert ToolInvokeMessage.MessageType.JSON in types @@ -182,8 +181,7 @@ def test_mcp_tool_invoke_handles_content_types_and_structured_output(sqlite_sess assert tool.latest_usage.total_tokens == 5 @staticmethod - @pytest.mark.parametrize("sqlite_session", [()], indirect=True) - def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(sqlite_session: Session): + def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(unbound_session: Session): tool = _build_mcp_tool() # Use model_construct to bypass pydantic validation and force unsupported resource path. bad_resource = EmbeddedResource.model_construct(type="resource", resource=object()) @@ -191,7 +189,7 @@ def test_mcp_tool_invoke_raises_for_unsupported_embedded_resource(sqlite_session with patch.object(MCPTool, "invoke_remote_mcp_tool", return_value=result): with pytest.raises(ToolInvokeError, match="Unsupported embedded resource type"): - list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={})) + list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={})) @staticmethod def test_mcp_tool_handle_none_parameter_filters_empty_values(): @@ -336,7 +334,7 @@ def test_invoke_skips_forwarding_outside_enterprise_edition(config_overrides): _meta=None, ) with patch.object(tool, "_inject_forwarded_identity") as inject: - with patch("services.tools.legacy_mcp_tools_manage_service.MCPToolManageService"): + with patch("services.tools.mcp_tools_manage_service.MCPToolManageService"): with patch("core.entities.mcp_provider.MCPProviderEntity.decrypt_server_url", return_value="u"): with patch("core.entities.mcp_provider.MCPProviderEntity.decrypt_headers", return_value={}): # Should not raise; should not call enterprise. diff --git a/api/tests/unit_tests/core/tools/plugin_tool/test_tool.py b/api/tests/unit_tests/core/tools/plugin_tool/test_tool.py index c13de6adb69669..aef597d4f99715 100644 --- a/api/tests/unit_tests/core/tools/plugin_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/plugin_tool/test_tool.py @@ -4,7 +4,6 @@ from unittest.mock import patch import httpx -import pytest from sqlalchemy.orm import Session import core.plugin.impl.base as plugin_client_module @@ -43,8 +42,7 @@ def _build_plugin_tool(*, has_runtime_parameters: bool) -> PluginTool: ) -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_plugin_tool_invoke_and_fork_runtime(sqlite_session: Session) -> None: +def test_plugin_tool_invoke_and_fork_runtime(unbound_session: Session) -> None: tool = _build_plugin_tool(has_runtime_parameters=False) requests: list[httpx.Request] = [] @@ -63,7 +61,7 @@ def handle_request(request: httpx.Request) -> httpx.Response: return_value={"converted": 1}, ), ): - messages = list(tool.invoke(session=sqlite_session, user_id="user-1", tool_parameters={"raw": 1})) + messages = list(tool.invoke(session=unbound_session, user_id="user-1", tool_parameters={"raw": 1})) assert [m.message.text for m in messages] == ["ok"] # pyrefly: ignore[missing-attribute] assert len(requests) == 1 diff --git a/api/tests/unit_tests/core/tools/test_tool_engine.py b/api/tests/unit_tests/core/tools/test_tool_engine.py index 7082e46e3f47ba..e9236085e72a68 100644 --- a/api/tests/unit_tests/core/tools/test_tool_engine.py +++ b/api/tests/unit_tests/core/tools/test_tool_engine.py @@ -1,14 +1,12 @@ from __future__ import annotations from collections.abc import Generator -from dataclasses import dataclass from typing import Any from unittest.mock import patch from uuid import uuid4 import pytest from sqlalchemy import select -from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from core.app.entities.app_invoke_entities import InvokeFrom @@ -31,13 +29,9 @@ ToolInvokeError, ToolParameterValidationError, ) -from core.tools.tool_engine import ToolEngine -from models.model import AppMode, Message, MessageFile - - -@dataclass(eq=False, repr=False) -class _DatabaseBinding: - engine: Engine +from models.model import App, AppMode, Conversation, Message, MessageFile +from services.tools.tool_engine import ToolEngine +from tests.helpers.unit.model_factories import make_app, make_conversation def _message() -> Message: @@ -85,7 +79,7 @@ def tool_provider_type(self) -> ToolProviderType: def _invoke( self, - session: Any, + session: Session, user_id: str, tool_parameters: dict[str, Any], conversation_id: str | None = None, @@ -247,21 +241,34 @@ def test_convert_tool_response_deduplicates_only_matching_json(): assert ToolEngine.tool_response_to_str(messages) == text + "result" + text + '{"c": 3}{"c": 3}' -@pytest.mark.parametrize("sqlite_session", [(MessageFile,)], indirect=True) -def test_create_message_files_and_invoke_generator(sqlite_engine: Engine, sqlite_session: Session): +@pytest.mark.parametrize("sqlite_session", [(App, Conversation, Message, MessageFile)], indirect=True) +def test_create_message_files_and_invoke_generator(sqlite_session: Session, *, app_records): binaries = [ ToolInvokeMessageBinary(mimetype="image/png", url="https://example.com/abc.png"), ToolInvokeMessageBinary(mimetype="audio/wav", url="https://example.com/def.wav"), ] agent_message = _message() - with patch("core.tools.tool_engine.db", _DatabaseBinding(sqlite_engine)): - ids = ToolEngine._create_message_files( - tool_messages=binaries, - agent_message=agent_message, - invoke_from=InvokeFrom.DEBUGGER, - user_id=str(uuid4()), - ) - + sqlite_session.add_all( + [ + make_app(app_id=agent_message.app_id, tenant_id="tenant-1"), + make_conversation( + conversation_id=agent_message.conversation_id, + app_id=agent_message.app_id, + inputs={}, + from_source="console", + ), + agent_message, + ] + ) + sqlite_session.commit() + ids = ToolEngine._create_message_files( + records=app_records, + tenant_id="tenant-1", + tool_messages=binaries, + agent_message=agent_message, + invoke_from=InvokeFrom.DEBUGGER, + user_id=str(uuid4()), + ) message_files = list(sqlite_session.scalars(select(MessageFile).order_by(MessageFile.created_at)).all()) assert ids == [message_file.id for message_file in message_files] assert len(message_files) == 2 @@ -274,13 +281,12 @@ def test_create_message_files_and_invoke_generator(sqlite_engine: Engine, sqlite assert invoked[-1].error is None -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_generic_invoke_success_and_error_paths(sqlite_session: Session): +def test_generic_invoke_success_and_error_paths(unbound_session: Session): tool = _build_tool() callback = DifyWorkflowCallbackHandler() response = list( ToolEngine.generic_invoke( - session=sqlite_session, + session=unbound_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -298,7 +304,7 @@ def test_generic_invoke_success_and_error_paths(sqlite_session: Session): with pytest.raises(RuntimeError, match="boom"): list( ToolEngine.generic_invoke( - session=sqlite_session, + session=unbound_session, tool=tool, tool_parameters={"x": 1}, user_id="u1", @@ -308,8 +314,7 @@ def test_generic_invoke_success_and_error_paths(sqlite_session: Session): ) -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_success(sqlite_session: Session): +def test_agent_invoke_success(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() @@ -317,13 +322,14 @@ def test_agent_invoke_success(sqlite_session: Session): with patch.object(ToolEngine, "_invoke", return_value=iter([tool.create_text_message("ok"), meta])): with patch( - "core.tools.tool_engine.ToolFileMessageTransformer.transform_tool_invoke_messages", + "services.tools.tool_engine.ToolFileMessageTransformer.transform_tool_invoke_messages", side_effect=lambda messages, **kwargs: messages, ): with patch.object(ToolEngine, "_extract_tool_response_binary_and_text", return_value=iter([])): with patch.object(ToolEngine, "_create_message_files", return_value=[]): result_text, message_files, result_meta = ToolEngine.agent_invoke( - session=sqlite_session, + session=unbound_session, + records=app_records, tool=tool, tool_parameters="hello", user_id="u1", @@ -338,15 +344,15 @@ def test_agent_invoke_success(sqlite_session: Session): assert result_meta.error is None -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_param_validation_error(sqlite_session: Session): +def test_agent_invoke_param_validation_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolParameterValidationError("bad-param")): error_text, files, error_meta = ToolEngine.agent_invoke( - session=sqlite_session, + session=unbound_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", @@ -361,8 +367,7 @@ def test_agent_invoke_param_validation_error(sqlite_session: Session): assert error_meta.error -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_engine_meta_error(sqlite_session: Session): +def test_agent_invoke_engine_meta_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() @@ -370,7 +375,8 @@ def test_agent_invoke_engine_meta_error(sqlite_session: Session): with patch.object(ToolEngine, "_invoke", side_effect=engine_error): error_text, files, error_meta = ToolEngine.agent_invoke( - session=sqlite_session, + session=unbound_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", @@ -407,15 +413,15 @@ def test_convert_tool_response_excludes_variable_messages(): assert "variable_name" not in result -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_agent_invoke_tool_invoke_error(sqlite_session: Session): +def test_agent_invoke_tool_invoke_error(unbound_session: Session, app_records): tool = _build_tool(with_llm_parameter=True) callback = DifyAgentCallbackHandler() message = _message() with patch.object(ToolEngine, "_invoke", side_effect=ToolInvokeError("invoke boom")): error_text, files, _ = ToolEngine.agent_invoke( - session=sqlite_session, + session=unbound_session, + records=app_records, tool=tool, tool_parameters={"a": 1}, user_id="u1", diff --git a/api/tests/unit_tests/core/tools/utils/test_dataset_retriever_tool.py b/api/tests/unit_tests/core/tools/utils/test_dataset_retriever_tool.py index 5af06f9c5872e8..a9fceb1dd2bbbb 100644 --- a/api/tests/unit_tests/core/tools/utils/test_dataset_retriever_tool.py +++ b/api/tests/unit_tests/core/tools/utils/test_dataset_retriever_tool.py @@ -1,189 +1,97 @@ -"""Unit tests for DatasetRetrieverTool behavior and retrieval wiring.""" +"""Dataset tools query injected records and preserve their dependencies when forked.""" -from __future__ import annotations - -from unittest.mock import patch +from dataclasses import dataclass, field +from uuid import uuid4 import pytest from sqlalchemy.orm import Session from core.app.app_config.entities import DatasetRetrieveConfigEntity -from core.app.apps.message_based_app_queue_manager import MessageBasedAppQueueManager from core.app.entities.app_invoke_entities import InvokeFrom -from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.rag.retrieval.dataset_retrieval import DatasetRetrieval -from core.tools.utils.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool as RetrievalTool -from core.tools.utils.dataset_retriever_tool import DatasetRetrieverTool -from models.model import AppMode - - -def _retrieve_config() -> DatasetRetrieveConfigEntity: - return DatasetRetrieveConfigEntity(retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE) - - -def _hit_callback() -> DatasetIndexToolCallbackHandler: - with patch("core.app.apps.base_app_queue_manager.redis_client.setex"): - queue_manager = MessageBasedAppQueueManager( - task_id="task", - user_id="u", - invoke_from=InvokeFrom.DEBUGGER, - conversation_id="conversation", - app_mode=AppMode.CHAT, - message_id="message", - ) - return DatasetIndexToolCallbackHandler( - queue_manager=queue_manager, app_id="app", message_id="message", user_id="u", invoke_from=InvokeFrom.DEBUGGER - ) +from core.tools.__base.tool_runtime import ToolRuntime +from models.dataset import Dataset, Document +from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository +from services.tools.dataset.tool import DatasetRetrieverTool +from tests.helpers.unit.model_factories import make_dataset -def _retrieval_tool() -> RetrievalTool: - return RetrievalTool( - name="dataset_tool", - description="desc", - tenant_id="tenant", - dataset_id="d1", - retrieve_config=_retrieve_config(), - inputs={}, - return_resource=False, - retriever_from="dev", - ) +@dataclass +class Retrieval: + calls: list = field(default_factory=list) + def retrieve_dataset(self, **kwargs): + self.calls.append(kwargs) + return f"result:{kwargs['query']}" -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_returns_empty_for_empty_dataset_ids(sqlite_session: Session) -> None: - # Arrange - retrieve_config = _retrieve_config() - # Act - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=[], - retrieve_config=retrieve_config, - return_resource=False, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=_hit_callback(), - user_id="u", - inputs={}, - ) - - # Assert - assert tools == [] +class NoResources: + def return_retriever_resource_info(self, _resources): + raise AssertionError("unexpected resources") -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_returns_empty_for_missing_retrieve_config(sqlite_session: Session) -> None: - # Arrange - dataset_ids = ["d1"] - - # Act +@pytest.fixture +def dataset_tools(sqlite_session: Session, sqlite_session_factory): + dataset_id = str(uuid4()) + sqlite_session.add( + make_dataset(dataset_id=dataset_id, tenant_id="tenant", provider="external", description="one\ntwo") + ) + sqlite_session.commit() + retrieval = Retrieval() + config = DatasetRetrieveConfigEntity(retrieve_strategy="multiple") tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, + records=KnowledgeRetrievalRepository(sqlite_session_factory), + retrieval=lambda: retrieval, tenant_id="tenant", - dataset_ids=dataset_ids, - retrieve_config=None, # type: ignore[arg-type] - return_resource=False, + app_id="app", + dataset_ids=[dataset_id], + retrieve_config=config, + return_resource=True, invoke_from=InvokeFrom.DEBUGGER, - hit_callback=_hit_callback(), - user_id="u", - inputs={}, + hit_callback=NoResources(), + user_id="user", + inputs={"x": 1}, ) - - # Assert - assert tools == [] - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_get_dataset_tools_builds_tool_and_restores_strategy(sqlite_session: Session) -> None: - # Arrange - retrieve_config = _retrieve_config() - retrieval_tool = _retrieval_tool() - - # Act - with patch.object(DatasetRetrieval, "to_dataset_retriever_tool", return_value=[retrieval_tool]): - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=["d1"], - retrieve_config=retrieve_config, - return_resource=True, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=_hit_callback(), - user_id="u", - inputs={"x": 1}, - ) - - # Assert - assert len(tools) == 1 - assert tools[0].entity.identity.name == "dataset_tool" - assert retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE + return tools, retrieval, config -def _build_dataset_tool(sqlite_session: Session) -> tuple[DatasetRetrieverTool, RetrievalTool]: - retrieval_tool = _retrieval_tool() - with patch.object(DatasetRetrieval, "to_dataset_retriever_tool", return_value=[retrieval_tool]): - tools = DatasetRetrieverTool.get_dataset_tools( - session=sqlite_session, - tenant_id="tenant", - dataset_ids=["d1"], - retrieve_config=_retrieve_config(), - return_resource=False, - invoke_from=InvokeFrom.DEBUGGER, - hit_callback=_hit_callback(), - user_id="u", - inputs={}, - ) - return tools[0], retrieval_tool - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_runtime_parameters_shape(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - params = tool.get_runtime_parameters() - - # Assert - assert len(params) == 1 - assert params[0].name == "query" - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_empty_query_behavior(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - empty_query = list(tool.invoke(session=sqlite_session, user_id="u", tool_parameters={})) - - # Assert - assert len(empty_query) == 1 - assert empty_query[0].message.text == "please input query" - - -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_query_invocation_result(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) - - # Act - with patch.object(RetrievalTool, "_run", return_value="result:hello") as retrieve: - result = list(tool.invoke(session=sqlite_session, user_id="u", tool_parameters={"query": "hello"})) - - # Assert - assert len(result) == 1 +def test_tool_metadata_and_fork_preserve_injected_retrieval(dataset_tools, unbound_session: Session): + tools, retrieval, config = dataset_tools + assert len(tools) == 1 + tool = tools[0] + assert tool.entity.description.llm == "onetwo" + assert [parameter.name for parameter in tool.get_runtime_parameters()] == ["query"] + assert config.retrieve_strategy == "multiple" + forked = tool.fork_tool_runtime(ToolRuntime(tenant_id="tenant")) + result = list(forked.invoke(session=unbound_session, user_id="user", tool_parameters={"query": "hello"})) assert result[0].message.text == "result:hello" - retrieve.assert_called_once_with(sqlite_session, "hello") + assert retrieval.calls[0]["inputs"] == {"x": 1} + assert retrieval.calls[0]["app_id"] == "app" -@pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_validate_credentials(sqlite_session: Session) -> None: - # Arrange - tool, _ = _build_dataset_tool(sqlite_session) +def test_empty_query_does_not_start_retrieval(dataset_tools, unbound_session: Session): + tools, retrieval, _ = dataset_tools + assert list(tools[0].invoke(session=unbound_session, user_id="user", tool_parameters={}))[0].message.text == ( + "please input query" + ) + assert not retrieval.calls - # Act - result = tool.validate_credentials(credentials={}, parameters={}, format_only=False) - # Assert - assert result is None +@pytest.mark.parametrize("sqlite_session", [(Dataset, Document)], indirect=True) +@pytest.mark.parametrize("case", ["no-datasets", "no-config", "foreign-tenant"]) +def test_tool_preparation_rejects_unavailable_datasets(sqlite_session, sqlite_session_factory, case): + sqlite_session.add(make_dataset(dataset_id="dataset", tenant_id="other", provider="external")) + sqlite_session.commit() + result = DatasetRetrieverTool.get_dataset_tools( + records=KnowledgeRetrievalRepository(sqlite_session_factory), + retrieval=Retrieval, + tenant_id="tenant", + app_id="app", + dataset_ids=[] if case == "no-datasets" else ["dataset"], + retrieve_config=None if case == "no-config" else DatasetRetrieveConfigEntity(retrieve_strategy="single"), + return_resource=False, + invoke_from=InvokeFrom.DEBUGGER, + hit_callback=NoResources(), + user_id="user", + inputs={}, + ) + assert result == [] diff --git a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py index d066c8fe771fb5..5472ee63679001 100644 --- a/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py +++ b/api/tests/unit_tests/core/tools/workflow_as_tool/test_tool.py @@ -1,17 +1,19 @@ """Unit tests for workflow-as-tool behavior with real SQLite ORM boundaries.""" import json +import threading import uuid from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import Any -from unittest.mock import patch +from typing import Any, cast +from unittest.mock import Mock, create_autospec, patch import pytest from flask import Flask -from sqlalchemy import Engine, inspect, select +from sqlalchemy import Engine, Table, inspect, select from sqlalchemy.orm import Session, sessionmaker +from core.app.apps.draft_variable_saver import DraftVariableSaverFactory from core.app.entities.app_invoke_entities import InvokeFrom from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.common_entities import I18nObject @@ -22,21 +24,21 @@ ToolParameter, ToolProviderType, ) -from core.tools.errors import ToolInvokeError -from core.tools.tool_manager import ToolManager -from core.tools.workflow_as_tool import tool as workflow_tool_module -from core.tools.workflow_as_tool.tool import WorkflowTool +from core.tools.errors import ToolInvokeError, ToolNotFoundError from graphon.enums import WorkflowExecutionStatus from graphon.file import FILE_MODEL_IDENTITY, File, FileTransferMethod, FileType from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole from models.base import TypeBase -from models.engine import db from models.enums import EndUserType from models.model import App, AppMode, EndUser from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowRun, WorkflowType -from services.workflow_run_agg import WorkflowRunAgg -from tests.helpers.unit.model_factories import make_account +from repositories.tools.workflow_repository import WorkflowToolRepository +from repositories.workflow.definition_repository import workflow_snapshot +from services.tools.tool_manager import ToolManager +from services.tools.workflow.tool import WorkflowTool +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.ports import WorkflowRuntime TENANT_ID = "00000000-0000-0000-0000-000000000001" OTHER_TENANT_ID = "00000000-0000-0000-0000-000000000002" @@ -70,14 +72,12 @@ def call(*_args: Any, **kwargs: Any) -> Any: @pytest.fixture def sqlite_tool_db( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, ) -> Iterator[SqliteToolDb]: """Bind service-owned sessions to SQLite.""" models = (App, Workflow, EndUser, Account, Tenant, TenantAccountJoin) - TypeBase.metadata.create_all(sqlite_engine, tables=[model.__table__ for model in models]) + TypeBase.metadata.create_all(sqlite_engine, tables=[cast(Table, model.__table__) for model in models]) session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - monkeypatch.setattr(workflow_tool_module.session_factory, "create_session", session_maker) with session_maker() as caller_session: yield SqliteToolDb(engine=sqlite_engine, session_maker=session_maker, caller_session=caller_session) @@ -168,7 +168,13 @@ def _persist_workflow(db: SqliteToolDb, *, version: str, workflow_id: str | None return workflow -def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", version: str = "1") -> WorkflowTool: +def _build_tool( + *, + tenant_id: str = "test_tool", + workflow_app_id: str = "app-1", + version: str = "1", + workflow_runtime: WorkflowRuntime, +) -> WorkflowTool: entity = ToolEntity( identity=ToolIdentity(author="test", name="test tool", label=I18nObject(en_US="test tool"), provider="test"), parameters=[], @@ -177,7 +183,9 @@ def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", ) runtime = ToolRuntime(tenant_id=tenant_id, invoke_from=InvokeFrom.EXPLORE) return WorkflowTool( - execution_driver=WorkflowRunAgg.run, + draft_variable_saver=Mock( + return_value=create_autospec(DraftVariableSaverFactory, instance=True, spec_set=True) + ), workflow_app_id=workflow_app_id, workflow_id="workflow-1", workflow_as_tool_id="wf-tool-1", @@ -185,12 +193,21 @@ def _build_tool(*, tenant_id: str = "test_tool", workflow_app_id: str = "app-1", workflow_call_depth=1, entity=entity, runtime=runtime, + workflow_runtime=workflow_runtime, + queries=workflow_runtime.tools, ) def test_legacy_workflow_tool_executes_with_the_upper_owned_driver( - sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + sqlite_session: Session, + *, + workflow_runtime: WorkflowRuntime, ) -> None: + from core.ops.ops_trace_manager import OpsTraceManager + tool_db = SqliteToolDb(sqlite_engine, sqlite_session_factory, sqlite_session) _persist_tenant(tool_db) user = _persist_end_user(tool_db) @@ -227,8 +244,7 @@ def test_legacy_workflow_tool_executes_with_the_upper_owned_driver( sqlite_session.add(provider) sqlite_session.commit() flask_app = Flask(__name__) - flask_app.config["SQLALCHEMY_DATABASE_URI"] = str(sqlite_engine.url) - db.init_app(flask_app) + monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda *_args: None) with flask_app.app_context(): tool = ToolManager.get_tool_runtime( @@ -238,7 +254,9 @@ def test_legacy_workflow_tool_executes_with_the_upper_owned_driver( tenant_id=TENANT_ID, user_id=user.id, invoke_from=InvokeFrom.SERVICE_API, - execution_driver=WorkflowRunAgg.run, + tool_providers=workflow_runtime.tool_providers, + workflow_queries=workflow_runtime.tools, + workflow_runtime=workflow_runtime, ) messages = list(tool.invoke(session=sqlite_session, user_id=user.id, tool_parameters={})) @@ -255,41 +273,39 @@ def test_legacy_workflow_tool_executes_with_the_upper_owned_driver( def test_workflow_tool_should_raise_tool_invoke_error_when_result_has_error_field( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure that WorkflowTool will throw a `ToolInvokeError` exception when `WorkflowAppGenerator.generate` returns a result with `error` key inside the `data` element. """ - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # needs to patch those methods to avoid database access. monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {"error": "oops"}}, ) with pytest.raises(ToolInvokeError) as exc_info: # WorkflowTool always returns a generator, so we need to iterate to # actually `run` the tool. - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) assert exc_info.value.args == ("oops",) def test_workflow_tool_rejects_paused_generator_result( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): - tool = _build_tool() - user = make_account(account_id="test_user") + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: + tool = _build_tool(workflow_runtime=workflow_runtime) + user = _persist_account(sqlite_tool_db) generate, generate_calls = _record_calls( return_value={ "data": { @@ -302,30 +318,35 @@ def test_workflow_tool_rejects_paused_generator_result( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) with pytest.raises(ToolInvokeError, match="requires Engine-managed container execution"): - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) assert len(generate_calls) == 1 -def test_legacy_tool_disables_human_input( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): - tool = _build_tool() +def test_workflow_tool_does_not_use_pause_state_config( + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: + """Ensure pause_state_config is passed as None.""" + tool = _build_tool(workflow_runtime=workflow_runtime) monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert call_kwargs["pause_state_config"] is None @@ -333,11 +354,10 @@ def test_legacy_tool_disables_human_input( def test_workflow_tool_passes_parent_trace_context_from_runtime( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure nested workflow runtime metadata is forwarded as parent trace context.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( parent_workflow_run_id="outer-workflow-run-1", parent_node_execution_id="outer-node-execution-1", @@ -346,13 +366,16 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["parent_trace_context"].model_dump() == { @@ -362,11 +385,10 @@ def test_workflow_tool_passes_parent_trace_context_from_runtime( def test_workflow_tool_passes_parent_trace_session_id( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure nested workflows inherit the parent observability session ID.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ ToolParameter.get_simple_instance( name="trace_session_id", @@ -380,13 +402,22 @@ def test_workflow_tool_passes_parent_trace_session_id( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {"trace_session_id": "user-input-session"})) + list( + tool.invoke( + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={"trace_session_id": "user-input-session"}, + ) + ) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["inputs"]["trace_session_id"] == "user-input-session" @@ -394,11 +425,10 @@ def test_workflow_tool_passes_parent_trace_session_id( def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure private trace context does not overwrite same-named workflow inputs.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.entity.parameters = [ ToolParameter.get_simple_instance( name="outer_workflow_run_id", @@ -421,17 +451,20 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) list( tool.invoke( - sqlite_session, - "test_user", - { + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={ "outer_workflow_run_id": "user-workflow-input", "outer_node_execution_id": "user-node-input", }, @@ -448,11 +481,10 @@ def test_workflow_tool_keeps_user_inputs_named_like_trace_runtime_keys( def test_workflow_tool_can_clear_parent_trace_context( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure reused WorkflowTool instances do not keep stale parent trace context.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_parent_trace_context( parent_workflow_run_id="outer-workflow-run-1", parent_node_execution_id="outer-node-execution-1", @@ -462,37 +494,42 @@ def test_workflow_tool_can_clear_parent_trace_context( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "parent_trace_context" not in call_kwargs["args"] def test_workflow_tool_can_clear_trace_session_id( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure reused WorkflowTool instances do not keep stale trace session IDs.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.set_trace_session_id("session-1") tool.clear_trace_session_id() monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "trace_session_id" not in call_kwargs["args"] @@ -509,34 +546,38 @@ def test_workflow_tool_can_clear_trace_session_id( ) def test_workflow_tool_omits_parent_trace_context_when_runtime_is_incomplete( monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, runtime_parameters: dict[str, Any], -): + sqlite_tool_db: SqliteToolDb, + *, + workflow_runtime: WorkflowRuntime, +) -> None: """Ensure incomplete runtime metadata does not leak parent trace context into generator args.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool.runtime.runtime_parameters = runtime_parameters monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) + monkeypatch.setattr("libs.login.current_user", lambda *args, **kwargs: None) - list(tool.invoke(sqlite_session, "test_user", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) call_kwargs = generate_calls[-1] assert "parent_trace_context" not in call_kwargs["args"] def test_workflow_tool_should_generate_variable_messages_for_outputs( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Test that WorkflowTool should generate variable messages when there are outputs""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # Mock workflow outputs mock_outputs = {"result": "success", "count": 42, "data": {"key": "value"}} @@ -545,79 +586,78 @@ def test_workflow_tool_should_generate_variable_messages_for_outputs( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {"outputs": mock_outputs}}, ) # Execute tool invocation - messages = list(tool.invoke(sqlite_session, "test_user", {})) + messages = list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) # Verify variable messages - variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + variable_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.VariableMessage)] assert len(variable_messages) == 3 # Verify content of each variable message - variable_dict = {msg.message.variable_name: msg.message.variable_value for msg in variable_messages} + variable_dict = {message.variable_name: message.variable_value for message in variable_messages} assert variable_dict["result"] == "success" assert variable_dict["count"] == 42 assert variable_dict["data"] == {"key": "value"} # Verify text message - text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + text_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.TextMessage)] assert len(text_messages) == 1 - assert json.loads(text_messages[0].message.text) == mock_outputs + assert json.loads(text_messages[0].text) == mock_outputs # Verify JSON message - json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + json_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.JsonMessage)] assert len(json_messages) == 1 - assert json_messages[0].message.json_object == mock_outputs + assert json_messages[0].json_object == mock_outputs def test_workflow_tool_should_handle_empty_outputs( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Test that WorkflowTool should handle empty outputs correctly""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) # needs to patch those methods to avoid database access. monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) # replace `WorkflowAppGenerator.generate` 's return value. monkeypatch.setattr( - "core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", lambda *args, **kwargs: {"data": {}}, ) # Execute tool invocation - messages = list(tool.invoke(sqlite_session, "test_user", {})) + messages = list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="test_user", tool_parameters={})) # Verify generated messages # Should contain: 0 variable messages + 1 text message + 1 JSON message = 2 messages assert len(messages) == 2 # Verify no variable messages - variable_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.VARIABLE] + variable_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.VariableMessage)] assert len(variable_messages) == 0 # Verify text message - text_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.TEXT] + text_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.TextMessage)] assert len(text_messages) == 1 - assert text_messages[0].message.text == "{}" + assert text_messages[0].text == "{}" # Verify JSON message - json_messages = [msg for msg in messages if msg.type == ToolInvokeMessage.MessageType.JSON] + json_messages = [msg.message for msg in messages if isinstance(msg.message, ToolInvokeMessage.JsonMessage)] assert len(json_messages) == 1 - assert json_messages[0].message.json_object == {} + assert json_messages[0].json_object == {} @pytest.mark.parametrize( @@ -631,31 +671,35 @@ def test_workflow_tool_should_handle_empty_outputs( ("dict_var", {"key": "value"}), ], ) -def test_create_variable_message(var_name, var_value): +def test_create_variable_message(var_name: str, var_value: object, *, workflow_runtime: WorkflowRuntime) -> None: """Create variable messages for multiple value types.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) message = tool.create_variable_message(var_name, var_value) assert message.type == ToolInvokeMessage.MessageType.VARIABLE + assert isinstance(message.message, ToolInvokeMessage.VariableMessage) assert message.message.variable_name == var_name assert message.message.variable_value == var_value assert message.message.stream is False -def test_create_file_message_should_include_file_marker(): +def test_create_file_message_should_include_file_marker(*, workflow_runtime: WorkflowRuntime) -> None: """Ensure file message includes marker and meta payload.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) file_obj = object() message = tool.create_file_message(file_obj) # type: ignore[arg-type] assert message.type == ToolInvokeMessage.MessageType.FILE + assert isinstance(message.message, ToolInvokeMessage.FileMessage) assert message.message.file_marker == "file_marker" assert message.meta == {"file": file_obj} -def test_resolve_user_from_database_falls_back_to_end_user(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_falls_back_to_end_user( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure worker context can resolve EndUser when Account is missing.""" _persist_tenant(sqlite_tool_db) end_user = _persist_end_user(sqlite_tool_db) @@ -665,31 +709,33 @@ def test_resolve_user_from_database_falls_back_to_end_user(sqlite_tool_db: Sqlit tenant_id=OTHER_TENANT_ID, ) - tool = _build_tool(tenant_id=TENANT_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_runtime=workflow_runtime) tool.runtime.invoke_from = InvokeFrom.SERVICE_API - resolved_user = tool._resolve_user_from_database(user_id=end_user.id) + resolved_user = tool._resolve_user(user_id=end_user.id) assert isinstance(resolved_user, EndUser) assert resolved_user.id == end_user.id assert resolved_user.tenant_id == TENANT_ID assert inspect(resolved_user).detached is True - assert tool._resolve_user_from_database(user_id=other_tenant_end_user.id) is None + assert tool._resolve_user(user_id=other_tenant_end_user.id) is None -def test_resolve_user_from_database_returns_none_when_no_tenant(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_returns_none_when_no_tenant( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Return None if tenant cannot be found in worker context.""" - tool = _build_tool(tenant_id=OTHER_TENANT_ID) + tool = _build_tool(tenant_id=OTHER_TENANT_ID, workflow_runtime=workflow_runtime) tool.runtime.invoke_from = InvokeFrom.SERVICE_API - resolved_user = tool._resolve_user_from_database(user_id="any") + resolved_user = tool._resolve_user(user_id="any") assert resolved_user is None -def test_workflow_tool_provider_type_and_fork_runtime(): +def test_workflow_tool_provider_type_and_fork_runtime(*, workflow_runtime: WorkflowRuntime) -> None: """Verify provider type and forked runtime behavior.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) assert tool.tool_provider_type() == ToolProviderType.WORKFLOW assert tool.latest_usage.total_tokens == 0 @@ -699,19 +745,19 @@ def test_workflow_tool_provider_type_and_fork_runtime(): assert forked.runtime.tenant_id == "tenant-2" -def test_derive_usage_from_top_level_usage_key(): +def test_derive_usage_from_top_level_usage_key() -> None: """Derive usage from top-level usage dict.""" usage = WorkflowTool._derive_usage_from_result({"usage": {"total_tokens": 12, "total_price": "0.2"}}) assert usage.total_tokens == 12 -def test_derive_usage_from_metadata_usage(): +def test_derive_usage_from_metadata_usage() -> None: """Derive usage from metadata usage dict.""" metadata_usage = WorkflowTool._derive_usage_from_result({"metadata": {"usage": {"total_tokens": 7}}}) assert metadata_usage.total_tokens == 7 -def test_derive_usage_from_totals(): +def test_derive_usage_from_totals() -> None: """Derive usage from top-level totals fields.""" totals_usage = WorkflowTool._derive_usage_from_result( {"total_tokens": "9", "total_price": "1.3", "currency": "USD"} @@ -720,52 +766,55 @@ def test_derive_usage_from_totals(): assert str(totals_usage.total_price) == "1.3" -def test_derive_usage_from_empty(): +def test_derive_usage_from_empty() -> None: """Default usage values when result is empty.""" empty_usage = WorkflowTool._derive_usage_from_result({}) assert empty_usage.total_tokens == 0 -def test_extract_usage_from_nested(): +def test_extract_usage_from_nested() -> None: """Extract nested usage dict from result payloads.""" nested = WorkflowTool._extract_usage_dict({"nested": [{"data": {"usage": {"total_tokens": 3}}}]}) assert nested == {"total_tokens": 3} def test_invoke_raises_when_user_not_found( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Raise ToolInvokeError when user resolution fails.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: None) with pytest.raises(ToolInvokeError, match="User not found"): - list(tool.invoke(sqlite_session, "missing", {})) + list(tool.invoke(session=sqlite_tool_db.caller_session, user_id="missing", tool_parameters={})) -def test_resolve_user_from_database_returns_account(sqlite_tool_db: SqliteToolDb): +def test_resolve_tenant_user_returns_account( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Resolve Account and set tenant in worker context.""" tenant = _persist_tenant(sqlite_tool_db) account = _persist_account(sqlite_tool_db) - tool = _build_tool(tenant_id=TENANT_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_runtime=workflow_runtime) - resolved = tool._resolve_user_from_database(user_id=account.id) + resolved = tool._resolve_user(user_id=account.id) assert isinstance(resolved, Account) assert resolved.id == account.id assert resolved.current_tenant_id == tenant.id assert inspect(resolved).detached is True -def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb): +def test_get_workflow_and_get_app_db_branches( + sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Cover workflow/app retrieval branches and error cases.""" app = _persist_app(sqlite_tool_db) specific_workflow = _persist_workflow(sqlite_tool_db, version="1") latest_workflow = _persist_workflow(sqlite_tool_db, version="2") _persist_workflow(sqlite_tool_db, version=Workflow.VERSION_DRAFT) - tool = _build_tool(tenant_id=TENANT_ID, workflow_app_id=APP_ID) + tool = _build_tool(tenant_id=TENANT_ID, workflow_app_id=APP_ID, workflow_runtime=workflow_runtime) latest = tool._get_workflow(APP_ID, "") specific = tool._get_workflow(APP_ID, "1") @@ -773,20 +822,22 @@ def test_get_workflow_and_get_app_db_branches(sqlite_tool_db: SqliteToolDb): assert latest.id == latest_workflow.id assert specific.id == specific_workflow.id + assert isinstance(latest, WorkflowSnapshot) + assert isinstance(specific, WorkflowSnapshot) + assert latest == workflow_snapshot(latest_workflow) + assert specific == workflow_snapshot(specific_workflow) assert resolved_app.id == app.id - assert inspect(latest).detached is True - assert inspect(specific).detached is True assert inspect(resolved_app).detached is True - with pytest.raises(ValueError, match="workflow not found"): + with pytest.raises(ToolNotFoundError, match="workflow not found"): tool._get_workflow(APP_ID, "missing") - with pytest.raises(ValueError, match="app not found"): + with pytest.raises(ToolNotFoundError, match="app not found"): tool._get_app("00000000-0000-0000-0000-000000000099") -def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch) -> WorkflowTool: +def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> WorkflowTool: """Build a WorkflowTool and stub merged runtime parameters for files/query.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) files_param = ToolParameter.get_simple_instance( name="files", llm_description="files", @@ -806,9 +857,9 @@ def _setup_transform_args_tool(monkeypatch: pytest.MonkeyPatch) -> WorkflowTool: return tool -def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): +def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> None: """Transform args into parameters and files payloads.""" - tool = _setup_transform_args_tool(monkeypatch) + tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) build_file_from_stored_mapping, build_file_calls = _record_calls( side_effect=[ File( @@ -829,7 +880,7 @@ def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): ] ) monkeypatch.setattr( - "core.tools.workflow_as_tool.tool.build_file_from_stored_mapping", + "services.tools.workflow.tool.build_file_from_stored_mapping", build_file_from_stored_mapping, ) @@ -867,9 +918,9 @@ def test_transform_args_valid_files(monkeypatch: pytest.MonkeyPatch): assert all(call["tenant_id"] == "test_tool" for call in build_file_calls) -def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch): +def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch, *, workflow_runtime: WorkflowRuntime) -> None: """Ignore invalid file entries while keeping params.""" - tool = _setup_transform_args_tool(monkeypatch) + tool = _setup_transform_args_tool(monkeypatch, workflow_runtime=workflow_runtime) invalid_params, invalid_files = tool._transform_args({"query": "hello", "files": [{"invalid": True}]}) assert invalid_params == {"query": "hello"} assert invalid_files == [] @@ -877,11 +928,10 @@ def test_transform_args_invalid_files(monkeypatch: pytest.MonkeyPatch): @pytest.mark.parametrize("empty_value", [None, "", [], [None], [""]]) def test_transform_args_normalizes_optional_files_parameter( - monkeypatch: pytest.MonkeyPatch, - empty_value: Any, -): + monkeypatch: pytest.MonkeyPatch, empty_value: Any, *, workflow_runtime: WorkflowRuntime +) -> None: """Pass optional workflow file-list inputs as an empty list when no files were provided.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( name="images", llm_description="images", @@ -898,11 +948,10 @@ def test_transform_args_normalizes_optional_files_parameter( def test_workflow_tool_invocation_normalizes_optional_files_parameter( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -): + monkeypatch: pytest.MonkeyPatch, sqlite_tool_db: SqliteToolDb, *, workflow_runtime: WorkflowRuntime +) -> None: """Ensure casted empty FILES values do not reach workflow input validation as [None].""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) images_param = ToolParameter.get_simple_instance( name="images", llm_description="images", @@ -914,21 +963,29 @@ def test_workflow_tool_invocation_normalizes_optional_files_parameter( monkeypatch.setattr(tool, "_get_app", lambda *args, **kwargs: None) monkeypatch.setattr(tool, "_get_workflow", lambda *args, **kwargs: None) - user = make_account(account_id="test_user") + user = _persist_account(sqlite_tool_db) monkeypatch.setattr(tool, "_resolve_user", lambda *args, **kwargs: user) generate, generate_calls = _record_calls(return_value={"data": {}}) - monkeypatch.setattr("core.app.apps.workflow.app_generator.WorkflowAppGenerator.generate", generate) + monkeypatch.setattr( + "services.workflow.execution.adapters.workflow.app_generator.WorkflowAppGenerator.generate", generate + ) - list(tool.invoke(sqlite_session, "test_user", {"images": None})) + list( + tool.invoke( + session=sqlite_tool_db.caller_session, + user_id="test_user", + tool_parameters={"images": None}, + ) + ) call_kwargs = generate_calls[-1] assert call_kwargs["args"]["inputs"]["images"] == [] -def test_extract_files(): +def test_extract_files(*, workflow_runtime: WorkflowRuntime) -> None: """Extract file outputs into result and file list.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) built_files = [ File( id="file-1", @@ -943,7 +1000,7 @@ def test_extract_files(): reference="upload-file-2", ), ] - with patch("core.tools.workflow_as_tool.tool.build_from_mapping", side_effect=built_files): + with patch("services.tools.workflow.tool.build_from_mapping", side_effect=built_files): outputs = { "attachments": [ { @@ -965,10 +1022,273 @@ def test_extract_files(): assert len(extracted_files) == 2 -def test_update_file_mapping(): +def test_update_file_mapping(*, workflow_runtime: WorkflowRuntime) -> None: """Map tool/local file transfer methods into output shape.""" - tool = _build_tool() + tool = _build_tool(workflow_runtime=workflow_runtime) tool_file = tool._update_file_mapping({"transfer_method": "tool_file", "related_id": "tool-1"}) assert tool_file["tool_file_id"] == "tool-1" local_file = tool._update_file_mapping({"transfer_method": "local_file", "related_id": "upload-1"}) assert local_file["upload_file_id"] == "upload-1" + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.EXPLORE]) +@pytest.mark.parametrize("entry", ["transform", "workflow"]) +def test_composed_tool_fork_runs_real_generator_with_injected_draft_saver( + monkeypatch: pytest.MonkeyPatch, + sqlite_tool_db: SqliteToolDb, + invoke_from: InvokeFrom, + entry: str, + workflow_runtime: WorkflowRuntime, + *, + workflow_queries: WorkflowToolRepository, +) -> None: + from flask import Flask + from sqlalchemy import select + + from core.app.apps import base_app_queue_manager + from core.app.entities.app_invoke_entities import UserFrom, build_dify_run_context + from core.ops.ops_trace_manager import TraceQueueManager + from extensions.application_services.workflow_variables import build_workflow_variable_service + from graphon.nodes import BuiltinNodeTypes + from graphon.nodes.tool.entities import ToolNodeData + from graphon.runtime import RuntimeState, VariablePool + from models.tools import WorkflowToolProvider + from models.workflow import WorkflowDraftVariable + from services.tools import tools_transform_service + from services.workflow.execution.adapters.node_factory import DifyGraphInitContext, DifyNodeFactory + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + account = _persist_account(sqlite_tool_db) + app = _persist_app(sqlite_tool_db) + workflow = _persist_workflow(sqlite_tool_db, version="1") + workflow.graph = json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ) + provider = WorkflowToolProvider( + name="nested_workflow", + label="Nested workflow", + icon="icon.svg", + app_id=app.id, + version="1", + user_id=account.id, + tenant_id=app.tenant_id, + description="", + parameter_configuration="[]", + ) + sqlite_tool_db.caller_session.add(provider) + sqlite_tool_db.caller_session.commit() + variables = build_workflow_variable_service(database_client=sqlite_tool_db.session_maker) + saver = variables.saver_factory + resolver = workflow_runtime.agent_bindings + if entry == "transform": + controller = tools_transform_service.ToolTransformService.workflow_provider_to_controller( + provider, + draft_variable_saver=saver, + workflow_runtime=workflow_runtime, + queries=workflow_queries, + ) + original = controller.get_tools(app.tenant_id)[0] + tool = original.fork_tool_runtime(ToolRuntime(tenant_id=app.tenant_id, invoke_from=invoke_from)) + else: + run_context = build_dify_run_context( + tenant_id=app.tenant_id, + app_id=app.id, + user_id=account.id, + user_from=UserFrom.ACCOUNT, + invoke_from=invoke_from, + ) + state = RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0) + factory = DifyNodeFactory.from_graph_init_context( + graph_init_context=DifyGraphInitContext( + workflow_id=workflow.id, graph_config=workflow.graph_dict, run_context=run_context, call_depth=0 + ), + runtime_state=state, + draft_variable_saver=saver, + agent_binding_resolver=resolver, + workflow_runtime=workflow_runtime, + ) + runtime = factory._tool_runtime + handle = runtime.get_runtime( + node_id="outer-tool-node", + node_data=ToolNodeData( + title="Nested workflow", + provider_id=provider.id, + provider_type="workflow", + provider_name=provider.name, + tool_name=provider.name, + tool_label=provider.label, + tool_configurations={}, + tool_parameters={}, + ), + variable_pool=state.variable_pool, + ) + tool = runtime._tool_from_handle(handle) + assert isinstance(tool, WorkflowTool) + assert tool._draft_variable_saver is saver + + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + generator_module, + "TraceQueueManager", + Mock(return_value=create_autospec(TraceQueueManager, instance=True, spec_set=True)), + ) + monkeypatch.setattr(WorkflowAppGenerator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + outputs = {"result": "nested result"} + + def handle_response( + self: WorkflowAppGenerator, *, draft_var_saver_factory: DraftVariableSaverFactory, **_kwargs: Any + ) -> dict[str, Any]: + draft_var_saver_factory(app.id, "nested-node", BuiltinNodeTypes.LLM, "nested-execution").save( + process_data=None, outputs=outputs + ) + return {"data": {"outputs": outputs}} + + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", handle_response) + monkeypatch.setattr( + generator_module.WorkflowAppGenerateResponseConverter, "convert", lambda *, response, invoke_from: response + ) + with Flask(__name__).app_context(): + messages = list(tool._invoke(session=sqlite_tool_db.caller_session, user_id=account.id, tool_parameters={})) + assert messages + with sqlite_tool_db.session_maker() as session: + saved = session.scalar(select(WorkflowDraftVariable).where(WorkflowDraftVariable.node_id == "nested-node")) + if invoke_from == InvokeFrom.DEBUGGER: + assert saved is not None + assert saved.user_id == account.id + assert saved.app_id == app.id + assert saved.get_value().value == outputs["result"] + else: + assert saved is None + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.WEB_APP, InvokeFrom.SERVICE_API]) +def test_nested_workflow_preserves_admitted_end_user_and_resolves_agent_binding( + sqlite_tool_db: SqliteToolDb, + workflow_runtime: WorkflowRuntime, + monkeypatch: pytest.MonkeyPatch, + invoke_from: InvokeFrom, + config_overrides: Callable[..., None], +) -> None: + """Exercise tool admission, generator, worker, runner and node construction across apps.""" + from flask import Flask + + from core.ops.ops_trace_manager import OpsTraceManager + from enums.agent import WorkflowAgentBindingType + from models.tools import WorkflowToolProvider + from services.tools.workflow.provider import WorkflowToolProviderController + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow_entry import WorkflowEntry + from tests.helpers.unit.model_factories import make_app + from tests.unit_tests.core.workflow.nodes.agent_v2.test_binding_resolver import _agent, _binding, _snapshot + + config_overrides(AGENT_BACKEND_USE_FAKE=True) + account = _persist_account(sqlite_tool_db) + child = _persist_app(sqlite_tool_db) + outer = make_app(app_id=str(uuid.uuid4()), tenant_id=child.tenant_id) + user = _persist_end_user(sqlite_tool_db) + user.app_id = outer.id + user.session_id = "customer-42" + workflow = _persist_workflow(sqlite_tool_db, version="1") + workflow.graph = json.dumps( + { + "nodes": [ + {"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}, + { + "id": "agent-node", + "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"}, + }, + ], + "edges": [{"source": "start", "target": "agent-node", "sourceHandle": "source", "targetHandle": "target"}], + } + ) + agent = _agent(tenant_id=child.tenant_id) + sqlite_tool_db.caller_session.add_all([outer, agent]) + sqlite_tool_db.caller_session.flush() + snapshot = _snapshot(tenant_id=child.tenant_id, agent_id=agent.id) + sqlite_tool_db.caller_session.add(snapshot) + sqlite_tool_db.caller_session.flush() + ids = {"tenant_id": child.tenant_id, "app_id": child.id, "workflow_id": workflow.id, "node_id": "agent-node"} + sqlite_tool_db.caller_session.add( + _binding( + ids=ids, + agent_id=agent.id, + snapshot_id=snapshot.id, + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + ) + ) + provider = WorkflowToolProvider( + tenant_id=child.tenant_id, + app_id=child.id, + user_id=account.id, + name="child", + label="Child", + icon="", + description="", + version="1", + parameter_configuration="[]", + ) + sqlite_tool_db.caller_session.add(provider) + sqlite_tool_db.caller_session.commit() + controller = WorkflowToolProviderController.from_db( + provider, + queries=workflow_runtime.tools, + workflow_runtime=workflow_runtime, + draft_variable_saver=None, + ) + tool = controller.get_tools(child.tenant_id)[0].fork_tool_runtime( + ToolRuntime(tenant_id=child.tenant_id, invoke_from=invoke_from) + ) + monkeypatch.setattr(OpsTraceManager, "get_ops_trace_instance", lambda *_args: None) + resolved: list[tuple[str, str]] = [] + + def probe_engine(entry: WorkflowEntry) -> Iterator[object]: + engine = entry.graph_engine + pool = engine.runtime_state.variable_pool + node = engine._graph.nodes["agent-node"] + assert isinstance(node, DifyAgentNode) + bundle = node._binding_resolver.resolve( + tenant_id=child.tenant_id, + app_id=child.id, + workflow_id=workflow.id, + node_id="agent-node", + ) + user_id = pool.get(["sys", "user_id"]) + assert user_id is not None + assert isinstance(user_id.value, str) + resolved.append((user_id.value, bundle.snapshot.id)) + return iter(()) + + monkeypatch.setattr(WorkflowEntry, "run", probe_engine) + + def execute_worker( + *, worker: threading.Thread, respond: Callable[[], object] + ) -> dict[str, dict[str, dict[str, str]]]: + del respond + worker.run() + assert resolved == [("customer-42", snapshot.id)] + return {"data": {"outputs": {"caller": resolved[0][0]}}} + + monkeypatch.setattr(generator_module, "generate_response", execute_worker) + with Flask(__name__).app_context(): + result = list( + tool._invoke( + session=sqlite_tool_db.caller_session, + user_id=user.id, + tool_parameters={}, + app_id=outer.id, + ) + ) + assert any(item.type == ToolInvokeMessage.MessageType.VARIABLE for item in result) + # Tool admission must still reject an external identity from another tenant. + foreign_user = _persist_end_user(sqlite_tool_db, end_user_id=str(uuid.uuid4()), tenant_id=OTHER_TENANT_ID) + with pytest.raises(ToolInvokeError, match="User not found"): + list( + tool._invoke( + session=sqlite_tool_db.caller_session, + user_id=foreign_user.id, + tool_parameters={}, + app_id=outer.id, + ) + ) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py b/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py index 0af40af9b002f8..07b643396fd3e5 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent/test_message_transformer.py @@ -34,7 +34,7 @@ def _message_stream(messages: list[ToolInvokeMessage]) -> Generator[ToolInvokeMe def _run_transform(messages: list[ToolInvokeMessage]) -> tuple[str, ArrayFileSegment]: events = list( - AgentMessageTransformer().transform( + AgentMessageTransformer(lambda _name, default: (default, None)).transform( messages=_message_stream(messages), tool_info={}, parameters_for_log={}, @@ -54,7 +54,7 @@ def _run_transform(messages: list[ToolInvokeMessage]) -> tuple[str, ArrayFileSeg def test_transform_passes_conversation_id_to_tool_file_message_transformer() -> None: messages = _message_stream([]) - transformer = AgentMessageTransformer() + transformer = AgentMessageTransformer(lambda _name, default: (default, None)) with patch.object(ToolFileMessageTransformer, "transform_tool_invoke_messages", return_value=iter(())) as transform: result = list( @@ -240,7 +240,7 @@ def test_transform_closes_think_tag_when_tool_log_interrupts() -> None: def test_transform_streams_close_tag_before_post_tool_text() -> None: events = list( - AgentMessageTransformer().transform( + AgentMessageTransformer(lambda _name, default: (default, None)).transform( messages=_message_stream( [ _text("need to search"), diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py index b368fa5c1334b3..d5ccfeb0c47bd8 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_agent_node.py @@ -2,7 +2,7 @@ from datetime import UTC, datetime from types import SimpleNamespace from typing import cast -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, create_autospec, patch from uuid import UUID, uuid4 import pytest @@ -22,7 +22,7 @@ ) from dify_agent.protocol.snapshot import SessionSnapshot from pydantic_ai.messages import PartDeltaEvent, TextPartDelta -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from clients.agent_backend import ( AgentBackendInternalEventType, @@ -35,14 +35,8 @@ ) from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContext, InvokeFrom, UserFrom from core.workflow.file_reference import build_file_reference -from core.workflow.nodes.agent_v2 import DifyAgentNode -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingBundle, WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.entities import DifyAgentNodeData from core.workflow.nodes.agent_v2.output_adapter import WorkflowAgentOutputAdapter -from core.workflow.nodes.agent_v2.runtime_request_builder import ( - WorkflowAgentRuntimeBuildContext, - WorkflowAgentRuntimeRequestBuilder, -) from core.workflow.nodes.agent_v2.session_store import ( StoredWorkflowAgentSession, WorkflowAgentSessionScope, @@ -75,14 +69,22 @@ DeclaredOutputType, WorkflowNodeJobConfig, ) +from models.agent_runtime_contracts import WorkflowAgentBindingBundle +from models.tool_runtime_contracts import WorkflowToolQueries +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from services.agent.workspace_service import AgentWorkspaceNotFoundError -from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode as ServiceDifyAgentNode +from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode +from services.workflow.execution.adapters.agent_v2.dify_tools_builder import WorkflowAgentDifyToolsBuilder +from services.workflow.execution.adapters.agent_v2.runtime_request_builder import ( + WorkflowAgentRuntimeBuildContext, + WorkflowAgentRuntimeRequestBuilder, +) @pytest.fixture(autouse=True) def _stub_model_context_window(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( - "core.workflow.nodes.agent_v2.runtime_request_builder.resolve_model_context_window", + "services.workflow.execution.adapters.agent_v2.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None, ) @@ -399,22 +401,17 @@ def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> Ca def _drain_event_stream(node: DifyAgentNode): - stream = node._consume_event_stream( + return node._consume_event_stream( "run-1", inputs={}, process_data={"workflow_agent_binding_id": "binding-1"}, metadata={"agent_backend": {}}, - stream_text=False, ) - while True: - try: - next(stream) - except StopIteration as stop: - return stop.value def _node( *, + tool_providers, scenario: FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS, session_store: FakeSessionStore | None = None, declared_outputs: list[dict[str, object]] | None = None, @@ -480,7 +477,13 @@ def is_accessible_file_mapping(self, *, file_id: str, tenant_id: str, transfer_m ), ), binding_resolver=binding_resolver, - runtime_request_builder=runtime_request_builder or WorkflowAgentRuntimeRequestBuilder(), + runtime_request_builder=runtime_request_builder + or WorkflowAgentRuntimeRequestBuilder( + dify_tools_builder=WorkflowAgentDifyToolsBuilder( + tool_providers=tool_providers, + workflow_queries=create_autospec(WorkflowToolQueries, instance=True), + ) + ), agent_backend_client=client, event_adapter=AgentBackendRunEventAdapter(), output_adapter=WorkflowAgentOutputAdapter(), @@ -509,8 +512,8 @@ def test_extract_variable_selector_to_variable_mapping_uses_frontend_agent_task_ } -def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(): - events = list(_node(agent_backend_client=PlainTextOutputBackendClient())._run()) +def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(tool_providers): + events = list(_node(tool_providers=tool_providers, agent_backend_client=PlainTextOutputBackendClient())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -526,9 +529,10 @@ def test_agent_node_run_maps_successful_agent_backend_run_to_node_result(): assert "output_type_check" not in agent_log -def test_agent_node_structured_success_preserves_text_and_checks_only_custom_outputs(): +def test_agent_node_structured_success_preserves_text_and_checks_only_custom_outputs(tool_providers): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "summary", "type": DeclaredOutputType.STRING}], agent_backend_client=FileOutputBackendClient( output_payload={"text": "hello agent", "summary": "Short summary"} @@ -553,9 +557,10 @@ def test_agent_node_structured_success_preserves_text_and_checks_only_custom_out } -def test_agent_node_structured_output_type_failure_stops_the_node(): +def test_agent_node_structured_output_type_failure_stops_the_node(tool_providers): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "summary", "type": DeclaredOutputType.STRING}], agent_backend_client=FileOutputBackendClient(output_payload={"summary": 42}), )._run() @@ -568,11 +573,11 @@ def test_agent_node_structured_output_type_failure_stops_the_node(): assert agent_log["output_failure_decision"] == "fail_node" -def test_agent_node_uses_resolved_backend_binding_before_backend_invocation() -> None: +def test_agent_node_uses_resolved_backend_binding_before_backend_invocation(tool_providers) -> None: client = FakeAgentBackendRunClient() store = FakeSessionStore(binding_id="binding-2", backend_binding_ref="backend-binding-2") - events = list(_node(agent_backend_client=client, session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, agent_backend_client=client, session_store=store)._run()) assert len(events) == 1 assert client.request is not None @@ -582,7 +587,7 @@ def test_agent_node_uses_resolved_backend_binding_before_backend_invocation() -> assert len(store.resolved_scopes) == 1 -def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution() -> None: +def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution(tool_providers) -> None: binding_resolver = FakeBindingResolver() store = FakeSessionStore() store.loaded_session = StoredWorkflowAgentSession( @@ -602,7 +607,7 @@ def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution( backend_binding_ref="backend-binding-1", session_snapshot=None, ) - node = _node(binding_resolver=binding_resolver, session_store=store) + node = _node(tool_providers=tool_providers, binding_resolver=binding_resolver, session_store=store) events = list(node._run()) @@ -613,7 +618,11 @@ def test_agent_node_resume_resolves_the_generation_from_the_persisted_execution( assert binding_resolver.calls[0]["conversation_id"] == "conversation-1" -def test_agent_node_workflow_tool_does_not_resolve_outer_conversation_participant(sqlite_session: Session) -> None: +def test_agent_node_workflow_tool_does_not_resolve_outer_conversation_participant( + sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], + tool_providers, +) -> None: binding_resolver = FakeBindingResolver() binding_resolver.agent.scope = AgentScope.WORKFLOW_ONLY binding_resolver.agent.source = AgentSource.WORKFLOW @@ -623,12 +632,17 @@ def test_agent_node_workflow_tool_does_not_resolve_outer_conversation_participan sqlite_session.commit() store = FakeSessionStore() node = _node( + tool_providers=tool_providers, binding_resolver=binding_resolver, session_store=store, workflow_tool_invocation_id="tool-call-1", ) with ( - patch.object(binding_resolver, "resolve", wraps=WorkflowAgentBindingResolver().resolve), + patch.object( + binding_resolver, + "resolve", + wraps=WorkflowAgentBindingResolver(sqlite_session_factory).resolve, + ), patch.object( WorkflowAgentWorkspaceStore, "load_active_participant", @@ -649,13 +663,13 @@ def test_agent_node_workflow_tool_does_not_resolve_outer_conversation_participan assert scope.workspace_owner.owner_id == "workflow-run-1" -def test_agent_node_maps_persisted_participant_lookup_error_to_node_failure() -> None: +def test_agent_node_maps_persisted_participant_lookup_error_to_node_failure(tool_providers) -> None: class _UnavailableParticipantStore(FakeSessionStore): def load_existing_node_execution_scope(self, **kwargs: object) -> WorkflowAgentSessionScope | None: del kwargs raise AgentWorkspaceNotFoundError("Workflow node participant Binding is unavailable") - events = list(_node(session_store=_UnavailableParticipantStore())._run()) + events = list(_node(tool_providers=tool_providers, session_store=_UnavailableParticipantStore())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -664,10 +678,15 @@ def load_existing_node_execution_scope(self, **kwargs: object) -> WorkflowAgentS assert result.error_type == "agent_workflow_node_runtime_error" -def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_builder() -> None: +def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_builder(tool_providers) -> None: store = FakeSessionStore() - request_builder = WorkflowAgentRuntimeRequestBuilder() - node = _node(session_store=store, runtime_request_builder=request_builder) + request_builder = WorkflowAgentRuntimeRequestBuilder( + dify_tools_builder=WorkflowAgentDifyToolsBuilder( + tool_providers=tool_providers, + workflow_queries=create_autospec(WorkflowToolQueries, instance=True), + ) + ) + node = _node(tool_providers=tool_providers, session_store=store, runtime_request_builder=request_builder) execution_id = node.execution_id with patch.object(request_builder, "build", wraps=request_builder.build) as build: @@ -684,15 +703,12 @@ def test_agent_node_passes_execution_id_to_session_store_and_runtime_request_bui assert context.node_execution_id == execution_id -def test_agent_node_run_streams_agent_message_delta_as_text_chunks(): - events = list(_node(agent_backend_client=AgentMessageDeltaBackendClient())._run()) +def test_agent_node_records_agent_message_delta_without_text_stream(tool_providers): + events = list(_node(tool_providers=tool_providers, agent_backend_client=AgentMessageDeltaBackendClient())._run()) - assert events[:2] == [ - StreamChunkEvent(selector=["agent-node", "text"], chunk="hello ", is_final=False), - StreamChunkEvent(selector=["agent-node", "text"], chunk="", is_final=True), - ] - assert len(events) == 3 - result = cast(StreamCompletedEvent, events[2]).node_run_result + assert not any(isinstance(event, StreamChunkEvent) for event in events) + assert len(events) == 1 + result = cast(StreamCompletedEvent, events[0]).node_run_result assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.outputs == {"text": "hello agent"} agent_backend = result.metadata[WorkflowNodeExecutionMetadataKey.AGENT_LOG]["agent_backend"] @@ -701,9 +717,10 @@ def test_agent_node_run_streams_agent_message_delta_as_text_chunks(): assert agent_backend["agent_message_delta_length"] == len("hello ") -def test_agent_node_structured_run_keeps_agent_message_delta_out_of_text_stream(): +def test_agent_node_structured_run_keeps_agent_message_delta_out_of_text_stream(tool_providers): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "summary", "type": DeclaredOutputType.STRING}], agent_backend_client=AgentMessageDeltaBackendClient(), )._run() @@ -715,9 +732,10 @@ def test_agent_node_structured_run_keeps_agent_message_delta_out_of_text_stream( assert agent_backend["agent_message_delta_count"] == 1 -def test_agent_node_default_value_plain_run_keeps_agent_message_delta_out_of_text_stream(): +def test_agent_node_default_value_plain_run_keeps_agent_message_delta_out_of_text_stream(tool_providers): events = list( _node( + tool_providers=tool_providers, error_strategy=ErrorStrategy.DEFAULT_VALUE, agent_backend_client=AgentMessageDeltaBackendClient(), )._run() @@ -731,7 +749,7 @@ def test_agent_node_default_value_plain_run_keeps_agent_message_delta_out_of_tex assert agent_backend["agent_message_delta_count"] == 1 -def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping(tool_providers): tool_reference = build_file_reference(record_id="tool-file-1") with patch( "core.workflow.nodes.agent_v2.output_adapter.build_from_mapping", @@ -739,6 +757,7 @@ def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping() ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "tool_file", "reference": tool_reference}} @@ -752,7 +771,7 @@ def test_agent_node_run_normalizes_declared_file_output_with_canonical_mapping() assert report.value.reference == tool_reference -def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonical_mapping(tool_providers): datasource_reference = build_file_reference(record_id="datasource-file-1") with patch( "core.workflow.nodes.agent_v2.output_adapter.build_from_mapping", @@ -763,6 +782,7 @@ def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonica ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "datasource_file", "reference": datasource_reference}} @@ -777,11 +797,12 @@ def test_agent_node_run_normalizes_declared_datasource_file_output_with_canonica assert report.value.reference == datasource_reference -def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonical_mapping(): +def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonical_mapping(tool_providers): remote_url = "https://example.com/report.pdf" events = list( _node( + tool_providers=tool_providers, declared_outputs=[{"name": "report", "type": DeclaredOutputType.FILE}], agent_backend_client=FileOutputBackendClient( output_payload={"report": {"transfer_method": "remote_url", "url": remote_url}} @@ -796,7 +817,7 @@ def test_agent_node_run_normalizes_declared_remote_url_file_output_with_canonica assert report.value.remote_url == remote_url -def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_mappings(): +def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_mappings(tool_providers): first_reference = build_file_reference(record_id="tool-file-1") second_reference = build_file_reference(record_id="tool-file-2") with patch( @@ -808,6 +829,7 @@ def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_map ): events = list( _node( + tool_providers=tool_providers, declared_outputs=[ { "name": "attachments", @@ -832,9 +854,11 @@ def test_agent_node_run_normalizes_declared_array_file_output_with_canonical_map assert [item.reference for item in attachments.value] == [first_reference, second_reference] -def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(): +def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(tool_providers): store = FakeSessionStore() - events = list(_node(scenario=FakeAgentBackendScenario.FAILED, session_store=store)._run()) + events = list( + _node(tool_providers=tool_providers, scenario=FakeAgentBackendScenario.FAILED, session_store=store)._run() + ) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -844,11 +868,11 @@ def test_agent_node_run_maps_failed_agent_backend_run_to_node_result(): assert store.saved[0][2] == SessionSnapshot(layers={}) -def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(): +def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(tool_providers): existing_snapshot = SessionSnapshot(layers={}) store = FakeSessionStore(snapshot=existing_snapshot) client = FakeAgentBackendRunClient() - node = _node(agent_backend_client=client, session_store=store) + node = _node(tool_providers=tool_providers, agent_backend_client=client, session_store=store) events = list(node._run()) @@ -862,13 +886,13 @@ def test_agent_node_saves_success_snapshot_and_reuses_existing_snapshot(): assert client.request.session_snapshot is existing_snapshot -def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_metadata(): +def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_metadata(tool_providers): """A DB-side write failure must not crash the node; it should set ``session_snapshot_persist_error`` in the agent_backend metadata so the incident is observable from the workflow_node_executions record.""" store = ExplodingSessionStore() - events = list(_node(session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, session_store=store)._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -879,7 +903,7 @@ def test_agent_node_run_when_session_store_save_raises_records_persist_error_in_ @pytest.mark.parametrize("failure_kind", ["backend", "transport"]) -def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kind: str) -> None: +def test_agent_node_snapshot_save_failure_preserves_original_failure(tool_providers, failure_kind: str) -> None: store = ExplodingSessionStore() client = ( FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario.FAILED) @@ -887,7 +911,7 @@ def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kin else FailingStreamBackendClient() ) - events = list(_node(agent_backend_client=client, session_store=store)._run()) + events = list(_node(tool_providers=tool_providers, agent_backend_client=client, session_store=store)._run()) result = cast(StreamCompletedEvent, events[0]).node_run_result assert result.status == WorkflowNodeExecutionStatus.FAILED @@ -903,11 +927,14 @@ def test_agent_node_snapshot_save_failure_preserves_original_failure(failure_kin @pytest.mark.parametrize("terminal_type", ["failed", "cancelled"]) -def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_write(terminal_type: str) -> None: +def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_write( + tool_providers, terminal_type: str +) -> None: store = FakeSessionStore() events = list( _node( + tool_providers=tool_providers, agent_backend_client=TerminalWithoutSnapshotBackendClient(terminal_type=terminal_type), session_store=store, )._run() @@ -918,7 +945,7 @@ def test_agent_node_terminal_without_snapshot_preserves_prior_session_without_wr assert store.saved == [] -def test_agent_node_unexpected_post_resolution_failure_keeps_binding_identity(): +def test_agent_node_unexpected_post_resolution_failure_keeps_binding_identity(tool_providers): class _FailingSessionStore(FakeSessionStore): def load_or_create_node_execution_session( self, @@ -929,7 +956,7 @@ def load_or_create_node_execution_session( del scope, home_snapshot_id raise RuntimeError("session store failed") - events = list(_node(session_store=_FailingSessionStore())._run()) + events = list(_node(tool_providers=tool_providers, session_store=_FailingSessionStore())._run()) assert len(events) == 1 result = cast(StreamCompletedEvent, events[0]).node_run_result @@ -943,9 +970,9 @@ def load_or_create_node_execution_session( } -def test_agent_node_cancels_backend_run_when_stream_fails(): +def test_agent_node_cancels_backend_run_when_stream_fails(tool_providers): client = FailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = _drain_event_stream(node) @@ -957,9 +984,9 @@ def test_agent_node_cancels_backend_run_when_stream_fails(): assert client.cancel_requests[0].reason == "event_stream_failed" -def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure() -> None: +def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure(tool_providers) -> None: client = FailingAfterStartedStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = _drain_event_stream(node) @@ -969,9 +996,9 @@ def test_agent_node_forwards_last_stream_cursor_when_cancelling_after_failure() assert client.cancel_after == ["cursor-1"] -def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event(): +def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event(tool_providers): client = EmptyStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = _drain_event_stream(node) @@ -981,9 +1008,9 @@ def test_agent_node_cancels_backend_run_when_stream_ends_without_terminal_event( assert client.cancel_requests[0].reason == "stream_ended_without_terminal_event" -def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(): +def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(tool_providers): client = GenericFailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) terminal, failure = _drain_event_stream(node) @@ -995,9 +1022,9 @@ def test_agent_node_cancels_backend_run_when_stream_raises_unexpected_error(): assert client.cancel_requests[0].reason == "event_stream_failed" -def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog): +def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(tool_providers, caplog): client = CancelFailingStreamBackendClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) node.runtime_state.graph_execution = SimpleNamespace(aborted=True) terminal, failure = _drain_event_stream(node) @@ -1011,9 +1038,9 @@ def test_agent_node_uses_graph_abort_reason_when_cancel_request_fails(caplog): assert "Failed to finish cancelling Workflow Agent backend run" in caplog.text -def test_agent_node_cancels_backend_run_for_unexpected_internal_event(): +def test_agent_node_cancels_backend_run_for_unexpected_internal_event(tool_providers): client = FakeAgentBackendRunClient() - node = _node(agent_backend_client=client) + node = _node(tool_providers=tool_providers, agent_backend_client=client) node._agent_backend_client.cancel_run_and_wait = MagicMock( # type: ignore[method-assign] return_value=RunCancelledEvent(run_id="run-1") ) @@ -1089,7 +1116,7 @@ def test_agent_node_records_only_trusted_knowledge_sources_and_deduplicates(): (True, 2, {"text": "hello", "switch": "route-1"}, "route-1"), ], ) -def test_agent_node_selects_the_configured_success_exit(enabled, count, output, expected_handle): +def test_agent_node_selects_the_configured_success_exit(tool_providers, enabled, count, output, expected_handle): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1099,7 +1126,11 @@ def test_agent_node_selects_the_configured_success_exit(enabled, count, output, } } ) - node = _node(binding_resolver=resolver, agent_backend_client=FileOutputBackendClient(output_payload=output)) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=FileOutputBackendClient(output_payload=output), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.edge_source_handle == expected_handle @@ -1117,6 +1148,7 @@ def test_agent_node_selects_the_configured_success_exit(enabled, count, output, ], ) def test_agent_routes_initialize_through_workflow_node_factory( + tool_providers, enabled: bool, error_strategy: ErrorStrategy | None, expected_execution_type: NodeExecutionType | None, @@ -1127,7 +1159,7 @@ def test_agent_routes_initialize_through_workflow_node_factory( config_overrides(AGENT_BACKEND_USE_FAKE=True) binding_resolver = FakeBindingResolver() - template = _node(binding_resolver=binding_resolver) + template = _node(tool_providers=tool_providers, binding_resolver=binding_resolver) factory = DifyNodeFactory( template.init_params, template.runtime_state, @@ -1150,12 +1182,12 @@ def test_agent_routes_initialize_through_workflow_node_factory( return node = factory.create_node(node_config) - assert isinstance(node, ServiceDifyAgentNode) + assert isinstance(node, DifyAgentNode) assert node.execution_type == expected_execution_type assert node.node_data.agent_output_routes.enabled is enabled -def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branches(): +def test_agent_routes_use_the_successful_retry_to_fan_out_and_skip_other_branches(tool_providers): from graphon.engine.scheduler import Scheduler from graphon.engine_events.traversal import GraphEdgeSkippedEvent, GraphEdgeTakenEvent from graphon.enums import NodeState @@ -1187,7 +1219,11 @@ def _events(self, run_id: str): }, } ) - node = _node(binding_resolver=resolver, agent_backend_client=RetryingBackendClient(output_payload={})) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=RetryingBackendClient(output_payload={}), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.SUCCEEDED assert result.outputs["score"] == 42 @@ -1222,7 +1258,7 @@ def _events(self, run_id: str): @pytest.mark.parametrize("output", ["plain text", {}, {"switch": None}, {"switch": 1}, {"switch": "unknown"}]) -def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(output): +def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(tool_providers, output): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1232,7 +1268,11 @@ def test_agent_node_rejects_invalid_route_selection_without_custom_outputs(outpu } } ) - node = _node(binding_resolver=resolver, agent_backend_client=FileOutputBackendClient(output_payload=output)) + node = _node( + tool_providers=tool_providers, + binding_resolver=resolver, + agent_backend_client=FileOutputBackendClient(output_payload=output), + ) result = list(node._run())[-1].node_run_result assert result.status == WorkflowNodeExecutionStatus.FAILED assert result.error_type == "output_route_selection_failed" @@ -1266,7 +1306,7 @@ def test_agent_route_conditions_load_variables_only_when_routing_is_enabled(enab assert mapping == expected -def test_disabled_routes_continue_through_graphon_default_values(): +def test_disabled_routes_continue_through_graphon_default_values(tool_providers): from graphon.engine.event.node_failure import NodeFailureHandler from graphon.engine.scheduler import Scheduler from graphon.engine_events.node import NodeRunExceptionEvent, NodeRunFailedEvent @@ -1275,7 +1315,7 @@ def test_disabled_routes_continue_through_graphon_default_values(): from graphon.node_events import NodeRunResult from graphon.runtime.execution import GraphExecution - node = _node(error_strategy=ErrorStrategy.DEFAULT_VALUE) + node = _node(tool_providers=tool_providers, error_strategy=ErrorStrategy.DEFAULT_VALUE) edges = {target: Edge(id=target, tail="agent-node", head=target) for target in ["next-a", "next-b"]} graph = Graph(root_node=node, nodes={"agent-node": node}, edges=edges, out_edges={"agent-node": list(edges)}) failure = NodeRunFailedEvent( @@ -1297,7 +1337,7 @@ def test_disabled_routes_continue_through_graphon_default_values(): assert [event.edge_id for event in traversed] == ["next-a", "next-b"] -def test_enabled_routes_reject_default_value_before_execution(): +def test_enabled_routes_reject_default_value_before_execution(tool_providers): resolver = FakeBindingResolver() resolver.binding.node_job_config = WorkflowNodeJobConfig.model_validate( { @@ -1308,4 +1348,4 @@ def test_enabled_routes_reject_default_value_before_execution(): } ) with pytest.raises(ValueError, match="default-value"): - _node(binding_resolver=resolver, error_strategy=ErrorStrategy.DEFAULT_VALUE) + _node(tool_providers=tool_providers, binding_resolver=resolver, error_strategy=ErrorStrategy.DEFAULT_VALUE) diff --git a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py index d6142285774fb8..8c2924337af91e 100644 --- a/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py +++ b/api/tests/unit_tests/core/workflow/nodes/agent_v2/test_binding_resolver.py @@ -1,15 +1,16 @@ from collections.abc import Iterator -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock, create_autospec from uuid import uuid4 import pytest from dify_agent.protocol.snapshot import SessionSnapshot -from sqlalchemy import event, inspect +from flask import Flask +from sqlalchemy import create_engine, event, inspect from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker -from sqlalchemy.sql import Executable +from sqlalchemy.sql import ClauseElement, Executable -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingError, WorkflowAgentBindingResolver from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope, WorkflowAgentWorkspaceStore +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowNodeExecutionStatus from models.agent import ( Agent, @@ -25,14 +26,145 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, WorkflowNodeJobConfig +from models.agent_runtime_contracts import WorkflowAgentBindingError from models.enums import CreatorUserRole from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver from services.agent.workspace_service import AgentWorkspaceService + +@pytest.fixture +def binding_resolver(sqlite_session_factory: sessionmaker[Session]) -> WorkflowAgentBindingResolver: + return WorkflowAgentBindingResolver(sqlite_session_factory) + + +@pytest.mark.parametrize("explicit_worker", [False, True]) +def test_node_binding_resolution_uses_injected_database_through_nested_factories( + app: Flask, monkeypatch: pytest.MonkeyPatch, binding_resolver: WorkflowAgentBindingResolver, explicit_worker: bool +) -> None: + from clients.agent_backend import AgentBackendRunClient + from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context + from enums import DeploymentEdition + from extensions.ext_application_services import build_application_services + from extensions.ext_redis import RedisClientWrapper + from graphon.runtime import RuntimeState, VariablePool + from models.base import TypeBase + from services.workflow.execution.adapters.agent_v2.agent_node import DifyAgentNode + from services.workflow.execution.adapters.node_factory import DifyGraphInitContext, DifyNodeFactory + + # The normal unit-test factory remains bound to its empty database. The + # configured application/worker uses a second database with these bindings. + engine = create_engine("sqlite://") + sessions = sessionmaker(bind=engine, expire_on_commit=False) + monkeypatch.setattr( + "clients.agent_backend.factory.create_agent_backend_run_client", + Mock(return_value=create_autospec(AgentBackendRunClient, instance=True, spec_set=True)), + ) + try: + TypeBase.metadata.create_all( + engine, tables=[TypeBase.metadata.tables[model.__tablename__] for model in RESOLVER_MODELS] + ) + ids = _resolve_ids() + with sessions.begin() as session: + agent = _agent(tenant_id=ids["tenant_id"]) + session.add(agent) + session.flush() + snapshot = _snapshot(tenant_id=ids["tenant_id"], agent_id=agent.id) + session.add(snapshot) + session.flush() + session.add( + _binding( + ids=ids, + agent_id=agent.id, + snapshot_id=snapshot.id, + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + ) + ) + with pytest.raises(WorkflowAgentBindingError, match="not found"): + binding_resolver.resolve(**ids) + + services = build_application_services( + database_client=sessions, + deployment_edition=DeploymentEdition.COMMUNITY, + initialization_password="", + redis=create_autospec(RedisClientWrapper, instance=True, spec_set=True), + ) + resolver = WorkflowAgentBindingResolver(sessions) if explicit_worker else services.workflow_agent_bindings + monkeypatch.delitem(app.extensions, "application_services", raising=False) + context = DifyGraphInitContext( + workflow_id=ids["workflow_id"], + graph_config={"nodes": [], "edges": []}, + call_depth=0, + run_context=build_dify_run_context( + tenant_id=ids["tenant_id"], + app_id=ids["app_id"], + user_id="account", + user_from=UserFrom.ACCOUNT, + invoke_from=InvokeFrom.DEBUGGER, + ), + ) + with app.app_context(): + factory = DifyNodeFactory.from_graph_init_context( + graph_init_context=context, + runtime_state=RuntimeState(workflow_id=ids["workflow_id"], variable_pool=VariablePool(), start_at=0), + agent_binding_resolver=resolver, + workflow_runtime=services.workflow_runtime, + ) + nested = factory.with_runtime_state( + RuntimeState(workflow_id=ids["workflow_id"], variable_pool=VariablePool(), start_at=0) + ) + for owner in (factory, nested): + node = owner.create_node( + {"id": ids["node_id"], "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}} + ) + assert isinstance(node, DifyAgentNode) + bundle = node._binding_resolver.resolve(**ids) + assert bundle.snapshot.id == snapshot.id + assert inspect(bundle.snapshot).detached + + # Single-node debugging must reach the same injected database, + # without consulting an installed Flask application container. + from services.workflow.execution.adapters.workflow_entry import WorkflowEntry + from tests.helpers.unit.model_factories import make_workflow + + node_config = { + "id": ids["node_id"], + "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"}, + } + workflow = make_workflow( + tenant_id=ids["tenant_id"], + app_id=ids["app_id"], + workflow_id=ids["workflow_id"], + graph={"nodes": [node_config], "edges": []}, + ) + monkeypatch.setattr(WorkflowEntry, "_run_node_with_layers", Mock()) + node, _ = WorkflowEntry.single_step_run( + workflow=workflow, + node_id=ids["node_id"], + user_id="account", + user_inputs={}, + variable_pool=VariablePool(), + agent_binding_resolver=resolver, + workflow_runtime=services.workflow_runtime, + ) + assert isinstance(node, DifyAgentNode) + assert node._binding_resolver.resolve(**ids).snapshot.id == snapshot.id + with pytest.raises(ValueError, match="injected binding resolver"): + WorkflowEntry.single_step_run( + workflow=workflow, + node_id=ids["node_id"], + user_id="account", + user_inputs={}, + variable_pool=VariablePool(), + workflow_runtime=services.workflow_runtime, + ) + finally: + engine.dispose() + + RESOLVER_MODELS = (WorkflowAgentNodeBinding, Agent, AgentConfigSnapshot, AgentConfigRevision) CHATFLOW_MODELS = ( *RESOLVER_MODELS, @@ -82,6 +214,7 @@ def _conversation_participant( @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) @pytest.mark.parametrize("binding_type", [WorkflowAgentBindingType.ROSTER_AGENT, WorkflowAgentBindingType.INLINE_AGENT]) def test_chatflow_keeps_participant_config_and_home_after_agent_update( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, binding_type: WorkflowAgentBindingType, @@ -105,7 +238,7 @@ def test_chatflow_keeps_participant_config_and_home_after_agent_update( sqlite_session.flush() participant = _conversation_participant(sqlite_session, ids=ids, binding=binding, snapshot=original) sqlite_session.commit() - resolver = WorkflowAgentBindingResolver() + resolver = binding_resolver assert resolver.resolve(**ids).snapshot.id == original.id published = _snapshot(tenant_id=ids["tenant_id"], agent_id=agent.id) @@ -177,6 +310,7 @@ def test_chatflow_keeps_participant_config_and_home_after_agent_update( @pytest.mark.parametrize("mismatch", ["tenant", "app", "conversation", "node", "agent", "retired"]) @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) def test_chatflow_pin_is_scoped_to_active_conversation_node_participant( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, mismatch: str, ) -> None: @@ -212,12 +346,13 @@ def test_chatflow_pin_is_scoped_to_active_conversation_node_participant( participant.status = AgentWorkingResourceStatus.RETIRED sqlite_session.commit() - assert WorkflowAgentBindingResolver().resolve(**ids).snapshot.id == current.id + assert binding_resolver.resolve(**ids).snapshot.id == current.id @pytest.mark.parametrize("version_kind", [AgentConfigVersionKind.DRAFT, AgentConfigVersionKind.SNAPSHOT]) @pytest.mark.parametrize("sqlite_session", [CHATFLOW_MODELS], indirect=True) def test_chatflow_never_falls_back_from_invalid_pinned_generation( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, version_kind: AgentConfigVersionKind, ) -> None: @@ -240,7 +375,7 @@ def test_chatflow_never_falls_back_from_invalid_pinned_generation( sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) expected = ( "agent_binding_generation_invalid" if version_kind == AgentConfigVersionKind.DRAFT @@ -346,6 +481,7 @@ def record_statement(execute_state: ORMExecuteState) -> None: @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_returns_detached_binding_bundle( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -363,7 +499,7 @@ def test_binding_resolver_returns_detached_binding_bundle( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.binding.id == binding.id assert bundle.agent.id == agent.id @@ -375,6 +511,7 @@ def test_binding_resolver_returns_detached_binding_bundle( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_uses_active_snapshot_for_roster_agent( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -397,7 +534,7 @@ def test_binding_resolver_uses_active_snapshot_for_roster_agent( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.snapshot.id == active_snapshot.id @@ -411,6 +548,7 @@ def test_binding_resolver_uses_active_snapshot_for_roster_agent( ) @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, orm_statements: list[Executable], binding_type: WorkflowAgentBindingType, @@ -437,19 +575,24 @@ def test_binding_resolver_uses_pinned_snapshot_for_existing_node_execution( ) sqlite_session.add(binding) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve( + bundle = binding_resolver.resolve( **ids, binding_id=binding.id, snapshot_id=pinned_snapshot.id, ) assert bundle.snapshot.id == pinned_snapshot.id - assert binding.id in orm_statements[0].compile().params.values() - assert pinned_snapshot.id in orm_statements[-1].compile().params.values() + first_statement = orm_statements[0] + last_statement = orm_statements[-1] + assert isinstance(first_statement, ClauseElement) + assert isinstance(last_statement, ClauseElement) + assert binding.id in first_statement.compile().params.values() + assert pinned_snapshot.id in last_statement.compile().params.values() @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -473,7 +616,7 @@ def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids, binding_id=binding.id, snapshot_id="") + binding_resolver.resolve(**ids, binding_id=binding.id, snapshot_id="") assert exc_info.value.error_code == "agent_config_snapshot_not_found" @@ -483,11 +626,12 @@ def test_binding_resolver_does_not_fallback_from_an_explicit_empty_snapshot( [("binding-1", None), (None, "snapshot-1")], ) def test_binding_resolver_rejects_half_pinned_generation( + binding_resolver: WorkflowAgentBindingResolver, binding_id: str | None, snapshot_id: str | None, ) -> None: with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve( + binding_resolver.resolve( **_resolve_ids(), binding_id=binding_id, snapshot_id=snapshot_id, @@ -498,6 +642,7 @@ def test_binding_resolver_rejects_half_pinned_generation( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_rejects_unpublished_roster_agent( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -521,7 +666,7 @@ def test_binding_resolver_rejects_unpublished_roster_agent( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" assert "not been published" in str(exc_info.value) @@ -529,6 +674,7 @@ def test_binding_resolver_rejects_unpublished_roster_agent( @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -565,7 +711,7 @@ def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot ) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" sqlite_session.add( @@ -579,21 +725,24 @@ def test_binding_resolver_requires_publish_provenance_for_active_roster_snapshot ) sqlite_session.commit() - bundle = WorkflowAgentBindingResolver().resolve(**ids) + bundle = binding_resolver.resolve(**ids) assert bundle.agent.id == agent.id assert bundle.snapshot.id == snapshot.id -def test_binding_resolver_raises_when_binding_missing() -> None: +def test_binding_resolver_raises_when_binding_missing( + binding_resolver: WorkflowAgentBindingResolver, +) -> None: with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**_resolve_ids()) + binding_resolver.resolve(**_resolve_ids()) assert exc_info.value.error_code == "agent_binding_not_found" @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_raises_when_agent_archived( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -609,13 +758,14 @@ def test_binding_resolver_raises_when_agent_archived( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_not_available" @pytest.mark.parametrize("sqlite_session", [RESOLVER_MODELS], indirect=True) def test_binding_resolver_raises_when_snapshot_missing( + binding_resolver: WorkflowAgentBindingResolver, sqlite_session: Session, ) -> None: ids = _resolve_ids() @@ -631,6 +781,6 @@ def test_binding_resolver_raises_when_snapshot_missing( sqlite_session.add(binding) sqlite_session.commit() with pytest.raises(WorkflowAgentBindingError) as exc_info: - WorkflowAgentBindingResolver().resolve(**ids) + binding_resolver.resolve(**ids) assert exc_info.value.error_code == "agent_config_snapshot_not_found" diff --git a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py index 0ba57ededc5872..3c72c205a76f26 100644 --- a/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py +++ b/api/tests/unit_tests/core/workflow/nodes/human_input_v2/test_node.py @@ -6,7 +6,6 @@ import pytest from pydantic import SecretStr, TypeAdapter -from core.app.apps.workflow_app_runner import WorkflowBasedAppRunner from core.app.entities.queue_entities import ( QueueHumanInputFormFilledEvent, QueueHumanInputFormTimeoutEvent, @@ -31,7 +30,6 @@ from core.workflow.nodes.human_input_v2.events import NodeRunHumanInputV2FormRequiredEvent from core.workflow.nodes.human_input_v2.node import HumanInputNode from core.workflow.nodes.human_input_v2.runtime import HumanInputDeliveryError, HumanInputRuntime, PreparedForm -from core.workflow.workflow_entry import WorkflowEntry from enums.human_input import ButtonStyle, HumanInputFormKind, HumanInputFormStatus, ValueSourceType from graphon.engine_events import ( EngineEvent, @@ -56,6 +54,8 @@ ) from repositories.human_input_v2.form_repository import Form, FormSubmission from services.tools.provider_queries import ToolProviderIcons +from services.workflow.execution.adapters.events import WorkflowEventPublisher +from services.workflow.execution.adapters.workflow_entry import WorkflowEntry from services.workflow.execution.ports import ExecutionContexts from tests.unit_tests.core.app.apps.common.test_workflow_response_converter import ( _workflow_response_converter_human_input_build_converter as _build_converter, @@ -147,11 +147,11 @@ def submit(form: Form) -> Form: def test_v1_event_filter_preserves_v2_completion_without_duplicate_events( form: Form, status: HumanInputFormStatus ) -> None: - from core.repositories.human_input_repository import HumanInputFormSubmissionRepository - from core.workflow.nodes.human_input.boundary import HumanInputFormEventFilter from graphon.engine.filter import EngineEventFilterContext, filter_engine_events from graphon.graph import Graph from graphon.runtime import ReadOnlyRuntimeStateWrapper + from repositories.human_input.form_repository import HumanInputFormSubmissionRepository + from services.workflow.execution.adapters.human_input_events import HumanInputFormEventFilter form = submit(form) if status == HumanInputFormStatus.SUBMITTED else replace(form, status=status) node, _ = build_node(form) @@ -280,11 +280,12 @@ def test_actual_graph_queue_response_path_preserves_order_and_fields( ) -> None: form = submit(form) if status == HumanInputFormStatus.SUBMITTED else replace(form, status=status) node, _ = build_node(form) - runner = create_autospec(WorkflowBasedAppRunner, instance=True) - entry = WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state) + queue_manager = MagicMock() + publisher = WorkflowEventPublisher(queue_manager, notify_pause=MagicMock()) + entry = WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state, human_form_reader=MagicMock()) for event in list(engine_events(node))[1:]: - WorkflowBasedAppRunner.handle_event(runner, entry, event) - queue_events = [call.args[0] for call in runner._publish_event.call_args_list] + publisher.publish(entry, event) + queue_events = [call.args[0] for call in queue_manager.publish.call_args_list] assert isinstance(queue_events[1], QueueNodeSucceededEvent) converter = _build_converter(workflow_contexts=workflow_contexts, tool_providers=tool_providers) converter.workflow_start_to_stream_response( @@ -311,12 +312,14 @@ def test_actual_graph_queue_response_path_preserves_order_and_fields( @pytest.mark.parametrize("token", ["initiator-token", None]) def test_dify_projection_serializes_the_runtime_token_without_a_graph_reason(form: Form, token: str | None) -> None: - from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter from core.app.entities.task_entities import HumanInputRequiredPauseReasonPayload + from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter prepared = PreparedForm(form, SecretStr(token) if token else None) with patch( - "core.app.apps.common.workflow_response_converter.Session", side_effect=AssertionError("unexpected lookup") + "services.workflow.execution.adapters.response_converter.Session", + side_effect=AssertionError("unexpected lookup"), + create=True, ): data = WorkflowResponseConverter.human_input_v2_required_data(prepared, node_id="node-1") assert data.form_token == token @@ -379,11 +382,12 @@ def test_file_segments_survive_node_graph_queue_and_response( assert isinstance(graph_events[2], NodeRunSucceededEvent) assert graph_events[1].rendered_content == "Answer: yes; files: [1 files]; attachment: [file]" assert isinstance(graph_events[2].node_run_result.outputs["attachment"], FileSegment) - runner = create_autospec(WorkflowBasedAppRunner, instance=True) - WorkflowBasedAppRunner.handle_event( - runner, WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state), graph_events[1] + queue_manager = MagicMock() + publisher = WorkflowEventPublisher(queue_manager, notify_pause=MagicMock()) + publisher.publish( + WorkflowEntry({}, Graph(root_node=node), 0, node.runtime_state, human_form_reader=MagicMock()), graph_events[1] ) - queue_event = runner._publish_event.call_args.args[0] + queue_event = queue_manager.publish.call_args.args[0] assert isinstance(queue_event.submitted_data["attachment"], FileSegment) assert isinstance(queue_event.submitted_data["files"], ArrayFileSegment) converter = _build_converter(workflow_contexts=workflow_contexts, tool_providers=tool_providers) @@ -439,7 +443,7 @@ def test_filled_rendering_skips_missing_and_null_inputs_and_preserves_select_tex def test_waiting_projection_freezes_select_options_and_deduplicates_slots(form: Form) -> None: - from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter + from services.workflow.execution.adapters.response_converter import WorkflowResponseConverter option = SelectInput(output_variable_name="choice", options=("old-a", "old-b"), default_value="old-b") form = replace(form, resolved_form=form.resolved_form.model_copy(update={"parts": (option, option)})) diff --git a/api/tests/unit_tests/core/workflow/test_node_runtime.py b/api/tests/unit_tests/core/workflow/test_node_runtime.py index 895b77eeddfac9..d40934cbf6ed20 100644 --- a/api/tests/unit_tests/core/workflow/test_node_runtime.py +++ b/api/tests/unit_tests/core/workflow/test_node_runtime.py @@ -2301,11 +2301,14 @@ def test_dify_tool_node_runtime_does_not_inject_outer_workflow_run_id_for_non_wo get_runtime.assert_called_once() @staticmethod - def test_dify_tool_node_runtime_builds_workflow_tool_container_payload(monkeypatch: pytest.MonkeyPatch) -> None: + def test_dify_tool_node_runtime_builds_workflow_tool_container_payload( + monkeypatch: pytest.MonkeyPatch, + workflow_runtime: WorkflowExecutionDependencies, + ) -> None: from core.tools.__base.tool_runtime import ToolRuntime from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolEntity, ToolIdentity - from core.tools.workflow_as_tool.tool import WorkflowTool + from services.tools.workflow.tool import WorkflowTool tool = WorkflowTool( workflow_app_id=str(uuid4()), @@ -2325,6 +2328,9 @@ def test_dify_tool_node_runtime_builds_workflow_tool_container_payload(monkeypat has_runtime_parameters=False, ), runtime=ToolRuntime(tenant_id="tenant-id", invoke_from=InvokeFrom.DEBUGGER), + queries=workflow_runtime.tools, + draft_variable_saver=None, + workflow_runtime=workflow_runtime, ) prepared_parameters: list[Mapping[str, object]] = [] @@ -2335,7 +2341,7 @@ def prepare_container_inputs( return {"amount": Decimal("1.25")}, [{"id": "file-id", "name": "input.txt"}] monkeypatch.setattr(tool, "prepare_container_inputs", prepare_container_inputs) - runtime = core_node_runtime.DifyToolNodeRuntime(_build_run_context()) + runtime = DifyToolNodeRuntime(_build_run_context(), workflow_runtime=workflow_runtime) payload = runtime.build_workflow_tool_container_payload( tool_runtime=ToolRuntimeHandle(raw=tool), @@ -2357,8 +2363,10 @@ def prepare_container_inputs( assert prepared_parameters == [{"amount": "1.25"}] @staticmethod - def test_dify_tool_node_runtime_rejects_non_workflow_tool_container_payload() -> None: - runtime = core_node_runtime.DifyToolNodeRuntime(_build_run_context()) + def test_dify_tool_node_runtime_rejects_non_workflow_tool_container_payload( + workflow_runtime: WorkflowExecutionDependencies, + ) -> None: + runtime = DifyToolNodeRuntime(_build_run_context(), workflow_runtime=workflow_runtime) with pytest.raises(ToolRuntimeResolutionError, match="resolved tool is not a Workflow Tool"): runtime.build_workflow_tool_container_payload( tool_runtime=ToolRuntimeHandle(raw=object()), diff --git a/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py b/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py index b44ce1f9c07d8f..7bfb61fe2c25b2 100644 --- a/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py +++ b/api/tests/unit_tests/core/workflow/test_workflow_tool_container_handler.py @@ -10,6 +10,7 @@ from threading import Event, get_ident from types import TracebackType from typing import override +from unittest.mock import create_autospec import pytest from flask import current_app, g @@ -41,25 +42,18 @@ from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolEntity, ToolIdentity from core.tools.workflow_as_tool.repository import WorkflowToolSource -from core.tools.workflow_as_tool.tool import WorkflowTool from core.workflow.node_execution_process_data import ( WORKFLOW_TOOL_INVOCATION_ID_KEY, ) -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.node_runtime import DifyToolFileManager, DifyToolNodeRuntime -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingResolver from core.workflow.nodes.human_input.boundary import resolve_human_input_node_id from core.workflow.nodes.human_input.callback import DifyHITLCallback from core.workflow.system_variables import SystemVariableKey, system_variable_selector from core.workflow.workflow_entry import WorkflowEntry -from core.workflow.workflow_tool_container_handler import ( - WorkflowToolContainerHandler, - WorkflowToolNestedContainerHandler, -) from core.workflow.workflow_tool_container_types import WorkflowToolContainerPayload from core.workflow.workflow_tool_node import DifyWorkflowToolNode from dify_app import DifyApp from enums import WorkflowKind +from extensions.application_services.workflow import WorkflowExecutionDependencies from extensions.storage.storage_type import StorageType from graphon.engine import Engine from graphon.engine.command import AbortCommand, InMemoryChannel @@ -139,7 +133,16 @@ from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.human_input_entities import HumanInputNodeData, UserActionConfig from models.model import AppMode +from models.tool_runtime_contracts import WorkflowToolQueries from models.workflow import WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom +from repositories.agent.runtime_repository import WorkflowAgentBindingResolver +from services.tools.workflow.tool import WorkflowTool +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.adapters.node_runtime import DifyToolFileManager, DifyToolNodeRuntime +from services.workflow.execution.adapters.workflow_tool_container_handler import ( + WorkflowToolContainerHandler, + WorkflowToolNestedContainerHandler, +) from services.workflow_persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer from services.workflow_run_index import WorkflowRunIndex from tests.helpers.unit.config_override import apply_config_overrides @@ -212,6 +215,8 @@ def _workflow_tool_container_workflow_tool_node( has_runtime_parameters=False, ), runtime=ToolRuntime(tenant_id="tenant", invoke_from=InvokeFrom.DEBUGGER), + queries=create_autospec(WorkflowToolQueries, instance=True, spec_set=True), + draft_variable_saver=None, ) monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr(tool, "prepare_container_inputs", lambda *_args, **_kwargs: (payload.inputs, [])) @@ -900,7 +905,9 @@ def execute(**_kwargs: object) -> dict[str, str]: raise CodeExecutionError("source code failed") return {"output": "retried output"} - monkeypatch.setattr("core.workflow.node_factory.CodeExecutor.execute_workflow_code_template", execute) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.CodeExecutor.execute_workflow_code_template", execute + ) source = _workflow_tool_container_workflow_tool_source() source_graph = dict(source.graph_config) source_graph["nodes"].insert( @@ -1045,7 +1052,7 @@ def build_human_input_callback( node_data: HumanInputNodeData, execution_id_getter: Callable[[], str | None], ) -> DifyHITLCallback: - human_input_app_ids.append(factory._human_input_runtime._run_context.app_id) + human_input_app_ids.append(factory.human_input_run_context.app_id) return DifyHITLCallback( form_repository=form_repository, node_data=node_data, @@ -1417,7 +1424,9 @@ def edge(source: str, target: str, handle: str = "source") -> dict[str, str]: return {"id": f"{source}-{target}", "source": source, "target": target, "sourceHandle": handle} tool_node, runtime, _ = _workflow_tool_event_delivery_workflow_tool_node() - monkeypatch.setattr("core.workflow.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime + ) outer_graph = { "nodes": [ node("start", "start", variables=[]), @@ -1661,9 +1670,10 @@ def listener_factory( def test_workflow_tool_agent_finds_its_persisted_caller_before_resolving_binding( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, caller_save_fails: bool, ) -> None: - binding_resolver = WorkflowAgentBindingResolver() + binding_resolver = WorkflowAgentBindingResolver(sqlite_session_factory) resolution_calls: list[dict[str, object]] = [] if caller_save_fails: @@ -1672,7 +1682,6 @@ def reject_resolution(**kwargs: object) -> None: pytest.fail("Binding resolution must wait for a persisted caller") monkeypatch.setattr(binding_resolver, "resolve", reject_resolution) - monkeypatch.setattr("core.workflow.node_factory.WorkflowAgentBindingResolver", lambda: binding_resolver) save = SQLAlchemyWorkflowNodeExecutionRepository.save def fail_caller_save( @@ -1759,8 +1768,15 @@ def fail_caller_save( index.seed_source(source.app_id, source.workflow_id, ()) layer.set_node_run_indices(index.indices) node.runtime_state.variable_pool.add(("sys", "workflow_run_id"), "outer-run") + graph = _workflow_tool_event_delivery_outer_graph(node) + graph.node_factory = DifyNodeFactory( + node.init_params, + node.runtime_state, + agent_binding_resolver=binding_resolver, + workflow_runtime=workflow_runtime, + ) engine = Engine( - graph=_workflow_tool_event_delivery_outer_graph(node), + graph=graph, runtime_state=node.runtime_state, command_channel=InMemoryChannel(), workers=1, @@ -1882,7 +1898,9 @@ def test_workflow_tool_dispatcher_enforces_file_ownership( start_at=1, ) template_node, runtime, payload = _workflow_tool_file_access_workflow_tool_node(runtime_state) - monkeypatch.setattr("core.workflow.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime) + monkeypatch.setattr( + "services.workflow.execution.adapters.node_factory.DifyToolNodeRuntime", lambda _, **_kwargs: runtime + ) tool_data = template_node.node_data.model_dump(mode="python") nodes = [ {"id": "start", "data": {"type": "start", "title": "Start", "variables": list[object]()}}, diff --git a/api/tests/unit_tests/dev/test_check_knowledge_fs_coverage.py b/api/tests/unit_tests/dev/test_check_knowledge_fs_coverage.py index 7d543f545165cd..0120135bc0a168 100644 --- a/api/tests/unit_tests/dev/test_check_knowledge_fs_coverage.py +++ b/api/tests/unit_tests/dev/test_check_knowledge_fs_coverage.py @@ -11,6 +11,7 @@ CoverageReport, CoverageSummary, collect_changed_glue_lines, + discover_knowledge_fs_production_paths, is_core_coverage_path, load_glue_coverage_paths, parse_added_lines, @@ -29,10 +30,10 @@ "api/controllers/inner_api/knowledge_fs/storage.py", "api/controllers/openapi/knowledge_fs.py", "api/controllers/service_api/knowledge_fs/resources.py", - "api/events/event_handlers/sync_knowledge_fs_bindings_when_app_published_workflow_updated.py", "api/extensions/ext_knowledge_fs_observability.py", "api/models/knowledge_fs.py", "api/repositories/sqlalchemy_knowledge_fs_cutover_repository.py", + "api/services/agent/knowledge_spaces.py", "api/services/dataset_knowledge_fs_upgrade_file_lease.py", "api/services/dataset_knowledge_fs_upgrade_service.py", "api/services/knowledge_fs/builtin_tool/knowledge_fs/knowledge_fs.py", @@ -122,6 +123,14 @@ def test_core_coverage_rejects_unclassified_knowledge_fs_production_files(tmp_pa validate_core_coverage(report, workspace_root=tmp_path, minimum=0) +def test_discovery_includes_cross_domain_knowledge_spaces_owner(tmp_path: Path) -> None: + owner_path = tmp_path / "api/services/agent/knowledge_spaces.py" + owner_path.parent.mkdir(parents=True) + owner_path.touch() + + assert discover_knowledge_fs_production_paths(tmp_path) == ("api/services/agent/knowledge_spaces.py",) + + @pytest.mark.parametrize( "migration", [ diff --git a/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py b/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py index d2e14d244a2ba5..374bd894a86968 100644 --- a/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py +++ b/api/tests/unit_tests/events/event_handlers/test_delete_tool_parameters_cache_when_sync_draft_workflow.py @@ -1,4 +1,5 @@ import logging +from types import SimpleNamespace import pytest @@ -55,6 +56,12 @@ def test_missing_tool_provider_does_not_log_error_traceback( }, ) monkeypatch.setattr(handler_module.ToolEntity, "model_validate", lambda _data: tool_entity) + tool_services = SimpleNamespace(tool_providers=object(), workflow_queries=object()) + monkeypatch.setattr( + handler_module, + "application_services", + lambda: SimpleNamespace(tools=tool_services), + ) monkeypatch.setattr( handler_module.ToolManager, "get_tool_runtime", diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_knowledge_fs_bindings_when_app_published_workflow_updated.py b/api/tests/unit_tests/events/event_handlers/test_sync_knowledge_fs_bindings_when_app_published_workflow_updated.py deleted file mode 100644 index 5a3d28c83aaced..00000000000000 --- a/api/tests/unit_tests/events/event_handlers/test_sync_knowledge_fs_bindings_when_app_published_workflow_updated.py +++ /dev/null @@ -1,135 +0,0 @@ -from __future__ import annotations - -import json -from collections.abc import Sequence -from dataclasses import dataclass, field -from typing import override - -import pytest -from sqlalchemy.orm import Session - -from events.event_handlers import sync_knowledge_fs_bindings_when_app_published_workflow_updated as handler -from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState -from models.model import App -from models.workflow import Workflow -from services.knowledge_fs.knowledge_retrieval_v2.validation import missing_control_space_ids - - -@dataclass -class _RecordingAppBindings: - calls: list[dict[str, object]] = field(default_factory=list) - - def sync_workflow_bindings(self, **kwargs: object) -> list[object]: - self.calls.append(kwargs) - return [] - - -@dataclass -class _Runtime: - app_bindings: _RecordingAppBindings - - -class _FlushRecordingSession(Session): - def __init__(self) -> None: - super().__init__() - self.flush_count = 0 - - @override - def flush(self, objects: Sequence[object] | None = None) -> None: - self.flush_count += 1 - super().flush(objects) - - -def _app() -> App: - return App(id="app-1", tenant_id="tenant-1") - - -def _workflow(*space_groups: list[str]) -> Workflow: - return Workflow( - created_by="account-1", - graph=json.dumps( - { - "nodes": [ - { - "data": { - "control_space_ids": spaces, - "query_variable_selector": ["start", "query"], - "title": "KnowledgeFS Retrieval", - "type": "knowledge-retrieval-v2", - } - } - for spaces in space_groups - ] - } - ), - ) - - -def test_extracts_deduplicated_control_spaces_in_graph_order() -> None: - workflow = _workflow(["space-a", "space-b"], ["space-b", "space-c"]) - - assert handler.get_control_space_ids_from_workflow(workflow) == ( - "space-a", - "space-b", - "space-c", - ) - - -def test_missing_space_validation_preserves_requested_order(sqlite_session: Session) -> None: - active_space = KnowledgeFSControlSpace( - tenant_id="tenant-1", - owner_account_id="account-1", - provisioning_key="provision-b", - knowledge_space_id="knowledge-space-b", - state=KnowledgeFSControlSpaceState.ACTIVE, - ) - active_space.id = "space-b" - sqlite_session.add(active_space) - sqlite_session.flush() - workflow = _workflow(["space-a", "space-b", "space-c"]) - - assert missing_control_space_ids( - session=sqlite_session, - tenant_id="tenant-1", - graph=workflow.graph_dict, - ) == ("space-a", "space-c") - - -def test_publish_handler_exactly_syncs_workflow_bindings(monkeypatch: pytest.MonkeyPatch) -> None: - bindings = _RecordingAppBindings() - monkeypatch.setattr(handler, "get_knowledge_fs_runtime", lambda _session_maker: _Runtime(bindings)) - workflow = _workflow(["space-a", "space-b"]) - publish_session = Session() - - handler.handle(_app(), published_workflow=workflow, session=publish_session) - - assert bindings.calls == [ - { - "tenant_id": "tenant-1", - "actor_account_id": "account-1", - "app_id": "app-1", - "control_space_ids": ["space-a", "space-b"], - "session": publish_session, - } - ] - - -def test_publish_union_includes_modern_agent_and_uses_the_publish_transaction(monkeypatch: pytest.MonkeyPatch) -> None: - bindings = _RecordingAppBindings() - monkeypatch.setattr(handler, "get_knowledge_fs_runtime", lambda _: _Runtime(bindings)) - publish_session = _FlushRecordingSession() - - def frozen_spaces(**kwargs: object) -> tuple[str, ...]: - assert publish_session.flush_count == 1 - assert kwargs["session"] is publish_session - return ("space-b", "space-c") - - monkeypatch.setattr(handler, "collect_workflow_agent_knowledge_space_ids", frozen_spaces) - workflow = _workflow(["space-a", "space-b"]) - graph = workflow.graph_dict - graph["nodes"].append({"data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}) - workflow.graph = json.dumps(graph) - handler.handle(_app(), published_workflow=workflow, session=publish_session) - assert len(bindings.calls) == 1 - assert bindings.calls[0]["control_space_ids"] == ["space-a", "space-b", "space-c"] - assert bindings.calls[0]["session"] is publish_session diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py b/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py deleted file mode 100644 index c5542161a254cc..00000000000000 --- a/api/tests/unit_tests/events/event_handlers/test_sync_plugin_trigger_when_app_created.py +++ /dev/null @@ -1,27 +0,0 @@ -from types import SimpleNamespace -from typing import cast -from unittest.mock import patch - -from events.app_event import app_draft_workflow_was_synced, app_published_workflow_was_updated -from events.event_handlers.sync_plugin_trigger_when_app_created import handle -from models.model import AppMode -from models.workflow import Workflow - - -def test_syncs_plugin_trigger_relationships_from_published_workflow() -> None: - app = SimpleNamespace(mode=AppMode.WORKFLOW.value) - published_workflow = cast(Workflow, object()) - - with patch( - "events.event_handlers.sync_plugin_trigger_when_app_created.TriggerService.sync_plugin_trigger_relationships" - ) as sync_relationships: - handle(app, published_workflow=published_workflow) - - sync_relationships.assert_called_once_with(app, published_workflow) - - -def test_draft_workflow_sync_does_not_register_plugin_trigger_handler() -> None: - sender = object() - - assert handle not in app_draft_workflow_was_synced.receivers_for(sender) - assert handle in app_published_workflow_was_updated.receivers_for(sender) diff --git a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py index 9f450474ec0aff..2a782e42dc4dc8 100644 --- a/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py +++ b/api/tests/unit_tests/events/event_handlers/test_sync_webhook_when_app_created.py @@ -1,32 +1,17 @@ -from types import SimpleNamespace -from typing import cast from unittest.mock import patch -from events.event_handlers import sync_webhook_when_app_created from events.event_handlers.sync_webhook_when_app_created import handle -from models.model import App -from models.workflow import Workflow +from models.model import AppMode +from tests.helpers.unit.model_factories import make_app, make_workflow def test_draft_sync_preserves_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) + app = make_app(mode=AppMode.WORKFLOW) + workflow = make_workflow() with patch( "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" ) as sync_webhook_relationships: handle(app, synced_draft_workflow=workflow) - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=False) - - -def test_published_sync_removes_stale_webhook_relationships() -> None: - app = cast(App, SimpleNamespace(mode="workflow")) - workflow = cast(Workflow, SimpleNamespace()) - - with patch( - "events.event_handlers.sync_webhook_when_app_created.WebhookService.sync_webhook_relationships" - ) as sync_webhook_relationships: - sync_webhook_when_app_created.handle_published(app, published_workflow=workflow) - - sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=True) + sync_webhook_relationships.assert_called_once_with(app, workflow, remove_stale=False, session_factory=None) diff --git a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py index e5ccc0aca6ba9b..f7bf706372f645 100644 --- a/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py +++ b/api/tests/unit_tests/extensions/otel/decorators/handlers/test_generate_handler.py @@ -6,11 +6,17 @@ 2. Verify span attribute mapping correctness """ -from unittest.mock import MagicMock +from unittest.mock import MagicMock, create_autospec + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from core.app.entities.app_invoke_entities import InvokeFrom from extensions.otel.decorators.handlers.generate_handler import AppGenerateHandler from extensions.otel.semconv import DifySpanAttributes, GenAIAttributes +from models import Account, App +from services.app.generation.runtime import AppGenerationRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.config_override import config_overrides_context @@ -19,8 +25,11 @@ class TestAppGenerateHandler: @config_overrides_context(ENABLE_OTEL=True) def test_compatible_with_real_function_signature( - self, tracer_provider_with_memory_exporter, mock_app_model, mock_account_user - ): + self, + tracer_provider_with_memory_exporter: TracerProvider, + mock_app_model: App, + mock_account_user: Account, + ) -> None: """ Verify handler compatibility with real AppGenerateService.generate signature. @@ -31,17 +40,18 @@ def test_compatible_with_real_function_signature( handler = AppGenerateHandler() - kwargs = { - "session": MagicMock(), - "app_model": mock_app_model, - "user": mock_account_user, - "args": {"workflow_id": "test-wf-123"}, - "invoke_from": InvokeFrom.DEBUGGER, - "streaming": True, - "root_node_id": None, - } - - arguments = handler._extract_arguments(AppGenerateService.generate, **kwargs) + arguments = handler._extract_arguments( + AppGenerateService.generate, + session=MagicMock(), + app_model=mock_app_model, + user=mock_account_user, + args={"workflow_id": "test-wf-123"}, + invoke_from=InvokeFrom.DEBUGGER, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + runtime=create_autospec(AppGenerationRuntime, instance=True, spec_set=True), + streaming=True, + root_node_id=None, + ) assert arguments is not None, "Failed to extract arguments from AppGenerateService.generate" assert "app_model" in arguments, "Handler uses app_model but parameter is missing" @@ -51,8 +61,12 @@ def test_compatible_with_real_function_signature( @config_overrides_context(ENABLE_OTEL=True) def test_all_span_attributes_set_correctly( - self, tracer_provider_with_memory_exporter, memory_span_exporter, mock_app_model, mock_account_user - ): + self, + tracer_provider_with_memory_exporter: TracerProvider, + memory_span_exporter: InMemorySpanExporter, + mock_app_model: App, + mock_account_user: Account, + ) -> None: """Verify all span attributes are mapped correctly""" handler = AppGenerateHandler() tracer = tracer_provider_with_memory_exporter.get_tracer(__name__) @@ -66,7 +80,14 @@ def test_all_span_attributes_set_correctly( mock_app_model.tenant_id = test_tenant_id mock_account_user.id = test_user_id - def dummy_func(app_model, user, args, invoke_from, streaming=True): + def dummy_func( + app_model: App, + user: Account, + args: dict[str, object], + invoke_from: InvokeFrom, + streaming: bool = True, + ) -> str: + del app_model, user, args, invoke_from, streaming return "result" handler.wrapper( @@ -82,6 +103,7 @@ def dummy_func(app_model, user, args, invoke_from, streaming=True): spans = memory_span_exporter.get_finished_spans() assert len(spans) == 1 attrs = spans[0].attributes + assert attrs is not None assert attrs[DifySpanAttributes.APP_ID] == test_app_id assert attrs[DifySpanAttributes.TENANT_ID] == test_tenant_id diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.py index e47ec96e9dfbbc..e4228085076562 100644 --- a/api/tests/unit_tests/extensions/test_ext_application_services.py +++ b/api/tests/unit_tests/extensions/test_ext_application_services.py @@ -62,7 +62,8 @@ from repositories.plugin_file_upload_repository import SQLAlchemyPluginFileUploadOwnerRepository from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository from repositories.upload_file_delivery_repository import UploadFileDeliveryQueryRepository -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from repositories.workflow.runtime_context_repository import WorkflowRuntimeContextRepository from repositories.workflow_run_archive_repository import WorkflowRunArchiveBundleQueryRepository from repositories.workspace.workspace_repository import WorkspaceRepository from services import audio_provider_gateway, recommended_app_catalog_gateway @@ -165,7 +166,7 @@ from services.web_authentication_service import WebAuthenticationService from services.webapp_access_adapters import EnterpriseWebAppAccessPolicyGateway from services.webapp_access_query_service import WebAppAccessQueryService, WebAppAccessUnavailableError -from services.workflow_app_log_query_service import WorkflowAppLogQueryService +from services.workflow.app_log_query_service import WorkflowAppLogQueryService from services.workflow_run_service import WorkflowRunService from services.workflow_statistic_query_service import WorkflowStatisticQueryService from tasks.delete_conversation_task import delete_conversation_related_data @@ -332,6 +333,12 @@ def test_build_application_services_preserves_composed_boundaries( assert isinstance(services.app_api_keys, AppApiKeyService) assert isinstance(services.dataset_api_keys, DatasetApiKeyService) assert services.dataset_api_keys._access is services.knowledge.datasets._dataset_access + assert services.snippet_generation._variables is services.workflow_variables + runtime_contexts = services.snippet_generation._runtime.contexts + assert isinstance(runtime_contexts, WorkflowRuntimeContextRepository) + assert runtime_contexts._sessions is sqlite_session_factory + assert services.knowledge.pipeline_generator.draft_variable_loader == services.workflow_variables.workflow_loader + assert services.knowledge.pipeline_generator.draft_variable_saver == services.workflow_variables.saver_factory assert isinstance(services.oauth_device, OAuthDeviceApplicationService) assert register_script.call_count == 3 assert services.conversation_queries._files is services.message_queries._files @@ -658,7 +665,7 @@ def test_build_application_services_wires_workflow_app_log_boundary( ) assert isinstance(services.workflow_app_logs, WorkflowAppLogQueryService) - assert isinstance(services.workflow_app_logs._logs, WorkflowAppLogQueryRepository) + assert isinstance(services.workflow_app_logs._logs, WorkflowAppLogRepository) assert services.workflow_app_logs._logs._session_factory is sqlite_session_factory diff --git a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py index de1c67cfa1553d..7cfc62099bed40 100644 --- a/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py +++ b/api/tests/unit_tests/extensions/test_workflow_warm_shutdown.py @@ -4,10 +4,10 @@ import pytest -from core.app.apps.workflow.active_workflow_tasks import reset_active_workflow_tasks -from core.app.apps.workflow.command_channels import CelerySignalCommandChannel from extensions import workflow_warm_shutdown from graphon.engine.command import AbortCommand +from services.workflow.execution.adapters.workflow.active_workflow_tasks import reset_active_workflow_tasks +from services.workflow.execution.adapters.workflow.command_channels import CelerySignalCommandChannel @pytest.fixture(autouse=True) diff --git a/api/tests/unit_tests/fields/test_workflow_fields.py b/api/tests/unit_tests/fields/test_workflow_fields.py new file mode 100644 index 00000000000000..71406b498ea151 --- /dev/null +++ b/api/tests/unit_tests/fields/test_workflow_fields.py @@ -0,0 +1,304 @@ +from dataclasses import asdict +from datetime import datetime + +import pytest +from pydantic import ValidationError +from sqlalchemy import event +from sqlalchemy.orm import Session + +from core.workflow.llm_environment_variable import LLMEnvironmentVariable +from extensions.application_services.snippets import build_snippet_service +from fields import workflow_fields +from graphon.variables import SecretVariable, StringVariable +from graphon.variables.variables import RAGPipelineVariable +from models.dataset import Pipeline +from models.snippet import CustomizedSnippet +from models.tools import WorkflowToolProvider +from models.workflow import WorkflowKind +from repositories.workflow.definition_repository import workflow_record +from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow_ref_service import WorkflowRef +from tests.helpers.unit.model_factories import make_account, make_workflow + + +@pytest.mark.parametrize("snippet", [False, True]) +def test_version_lists_share_publication_order_and_owner_scope(sqlite_session: Session, snippet: bool) -> None: + kind = WorkflowKind.SNIPPET if snippet else WorkflowKind.STANDARD + older = make_workflow(workflow_id="older", app_id="owner", version="z-old", kind=kind) + newer = make_workflow(workflow_id="newer", app_id="owner", version="a-new", kind=kind, marked_name="Release") + draft = make_workflow(workflow_id="draft", app_id="owner", kind=kind) + foreign_tenant = make_workflow( + workflow_id="foreign", app_id="owner", tenant_id="other", version="foreign", kind=kind + ) + foreign_owner = make_workflow(workflow_id="other-owner", app_id="other-owner", version="other", kind=kind) + for year, workflow in [(2020, draft), (2021, older), (2022, newer), (2023, foreign_tenant), (2024, foreign_owner)]: + workflow.created_at = datetime(year, 1, 1) + sqlite_session.add(workflow) + if snippet: + sqlite_session.add(make_workflow(workflow_id="wrong-kind", app_id="owner", version="wrong-kind")) + sqlite_session.commit() + + if snippet: + owner = CustomizedSnippet(id="owner", tenant_id="tenant-1", workflow_id=newer.id) + service = build_snippet_service(session=sqlite_session) + page, has_more = service.get_all_published_workflows(session=sqlite_session, snippet=owner, page=1, limit=1) + assert [item.id for item in page] == ["newer"] + assert has_more + page, has_more = service.get_all_published_workflows(session=sqlite_session, snippet=owner, page=2, limit=1) + assert [item.id for item in page] == ["older"] + else: + pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", workflow_id=newer.id) + pipeline.id = "owner" + pipelines = RagPipelineService(sqlite_session) + page, has_more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=2, user_id=None + ) + assert [item.id for item in page] == ["draft", "newer"] + assert has_more + page, has_more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=2, limit=2, user_id=None + ) + assert [item.id for item in page] == ["older"] + named, more = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=2, user_id="account-1", named_only=True + ) + assert [item.id for item in named] == ["newer"] + assert not more + assert not has_more + + +@pytest.fixture(autouse=True) +def _identity_workflow_encryption(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep variable serialization focused on ORM behavior, not the external key provider.""" + + monkeypatch.setattr(workflow_fields.encrypter, "encrypt_token", lambda *, token, **_kwargs: token) + monkeypatch.setattr(workflow_fields.encrypter, "decrypt_token", lambda *, token, **_kwargs: token) + + +def test_pipeline_variable_response_accepts_legacy_file_field_names() -> None: + response = workflow_fields.PipelineVariableResponse.model_validate( + { + "label": "Query", + "variable": "query", + "type": "single-file", + "belong_to_node_id": "shared", + "max_length": 0, + "required": False, + "unit": "", + "default_value": "", + "options": [], + "placeholder": "", + "tooltips": "", + "allowed_file_types": [], + "allow_file_extension": [".txt"], + "allow_file_upload_methods": ["remote_url"], + } + ).model_dump(mode="json") + + assert response["allowed_file_extensions"] == [".txt"] + assert response["allowed_file_upload_methods"] == ["remote_url"] + + +def test_pipeline_variable_response_accepts_explicit_null_optional_fields() -> None: + pipeline_variable = RAGPipelineVariable.model_validate( + { + "label": "Query", + "variable": "query", + "type": "text-input", + "belong_to_node_id": "shared", + "max_length": None, + "unit": None, + "default_value": None, + "options": None, + "placeholder": None, + "tooltips": None, + "allowed_file_types": None, + "allowed_file_extensions": None, + "allowed_file_upload_methods": None, + } + ).model_dump(mode="json") + + response = workflow_fields.PipelineVariableResponse.model_validate(pipeline_variable).model_dump(mode="json") + + assert response["max_length"] is None + assert response["allowed_file_types"] is None + assert response["allowed_file_extensions"] is None + assert response["allowed_file_upload_methods"] is None + + +def test_workflow_response_masks_secret_environment_variables(sqlite_session: Session) -> None: + workflow = make_workflow( + environment_variables=[ + SecretVariable(id="env-secret", name="API_KEY", value="plain-token", selector=["env", "API_KEY"]), + StringVariable(id="env-string", name="REGION", value="us-east-1", selector=["env", "REGION"]), + ] + ) + sqlite_session.add_all([make_account(), workflow]) + sqlite_session.commit() + + record = workflow_record(workflow, sqlite_session) + sqlite_session.close() + response = workflow_fields.WorkflowResponse.model_validate( + record, + from_attributes=True, + ).model_dump(mode="json") + + assert response["environment_variables"] == [ + { + "id": "env-secret", + "name": "API_KEY", + "value": workflow_fields.encrypter.full_mask_token(), + "value_type": "secret", + "description": "", + }, + { + "id": "env-string", + "name": "REGION", + "value": "us-east-1", + "value_type": "string", + "description": "", + }, + ] + + +def test_workflow_response_preserves_llm_environment_variable_type(sqlite_session: Session) -> None: + workflow = make_workflow( + environment_variables=[ + LLMEnvironmentVariable( + id="env-llm", + name="for_summarize", + value={"provider": "provider", "name": "model", "mode": "chat"}, + selector=["env", "for_summarize"], + ) + ] + ) + sqlite_session.add_all([make_account(), workflow]) + sqlite_session.commit() + + record = workflow_record(workflow, sqlite_session) + sqlite_session.close() + response = workflow_fields.WorkflowResponse.model_validate( + record, + from_attributes=True, + ).model_dump(mode="json") + + assert response["environment_variables"] == [ + { + "id": "env-llm", + "name": "for_summarize", + "value": {"provider": "provider", "name": "model", "mode": "chat"}, + "value_type": "llm", + "description": "", + } + ] + + +def test_workflow_response_rejects_invalid_environment_variable_dict(sqlite_session: Session) -> None: + data = asdict(workflow_record(make_workflow(), sqlite_session)) + data["environment_variables"] = [{"value_type": "not-a-segment-type"}] + sqlite_session.close() + with pytest.raises(ValidationError): + workflow_fields.WorkflowResponse.model_validate(data) + + +@pytest.mark.parametrize("owner", ["snippet", "pipeline"]) +@pytest.mark.parametrize("operation", ["draft", "published", "list", "update"]) +def test_workflow_records_serialize_after_session_closes( + sqlite_session: Session, + owner: str, + operation: str, +) -> None: + workflow = make_workflow(workflow_id="workflow-1", app_id="owner-1") + workflow.kind = WorkflowKind.SNIPPET if owner == "snippet" else WorkflowKind.STANDARD + workflow.version = "draft" if operation == "draft" else "2026-10-02 12:00:00" + workflow.updated_by = "editor-1" + author = make_account() + editor = make_account(account_id="editor-1", email="editor@example.com") + sqlite_session.add_all( + [ + workflow, + author, + editor, + WorkflowToolProvider( + name="workflow-tool", + label="Workflow", + icon="", + app_id="owner-1", + version=workflow.version, + user_id=author.id, + tenant_id=workflow.tenant_id, + description="", + ), + ] + ) + sqlite_session.commit() + if owner == "snippet": + snippet = CustomizedSnippet( + id="owner-1", + tenant_id="tenant-1", + name="Snippet", + description="", + type="node", + created_by=author.id, + workflow_id=workflow.id, + ) + snippets = build_snippet_service(session=sqlite_session) + match operation: + case "draft": + record = snippets.get_draft_workflow_record(snippet) + case "published": + record = snippets.get_published_workflow_record(snippet) + case "list": + records, _ = snippets.get_all_published_workflows( + session=sqlite_session, snippet=snippet, page=1, limit=20 + ) + record = records[0] + case _: + record = snippets.update_workflow( + session=sqlite_session, + snippet=snippet, + workflow_id=workflow.id, + account=editor, + data={"marked_name": "Updated"}, + ) + else: + pipeline = Pipeline(tenant_id="tenant-1", name="Pipeline", description="") + pipeline.id = "owner-1" + pipeline.workflow_id = workflow.id + pipelines = RagPipelineService(sqlite_session) + match operation: + case "draft": + record = pipelines.get_draft_workflow_record(pipeline) + case "published": + record = pipelines.get_published_workflow_record(pipeline) + case "list": + records, _ = pipelines.get_all_published_workflow( + session=sqlite_session, pipeline=pipeline, page=1, limit=20, user_id=None + ) + record = records[0] + case _: + record = pipelines.update_workflow( + session=sqlite_session, + account_id=editor.id, + data={"marked_name": "Updated"}, + workflow_ref=WorkflowRef(tenant_id="tenant-1", owner_id="owner-1", workflow_id=workflow.id), + ) + assert record is not None + engine = sqlite_session.get_bind() + sqlite_session.commit() + sqlite_session.close() + + def unexpected_query(*_args: object) -> None: + pytest.fail("workflow serialization must not execute SQL") + + event.listen(engine, "before_cursor_execute", unexpected_query) + try: + response = workflow_fields.WorkflowResponse.model_validate(record, from_attributes=True).model_dump(mode="json") + finally: + event.remove(engine, "before_cursor_execute", unexpected_query) + assert response["created_by"]["id"] == "account-1" + assert response["updated_by"]["id"] == "editor-1" + assert response["tool_published"] is True + assert response["graph"] == record.graph + if operation == "update": + assert response["marked_name"] == "Updated" diff --git a/api/tests/unit_tests/libs/test_schedule_utils.py b/api/tests/unit_tests/libs/test_schedule_utils.py index d1b9b0a84371e8..e47cae23f2ea3a 100644 --- a/api/tests/unit_tests/libs/test_schedule_utils.py +++ b/api/tests/unit_tests/libs/test_schedule_utils.py @@ -293,10 +293,10 @@ def test_execution_time_calculator_pattern(self): assert local_time.minute == 30 assert local_time.second == 0 - def test_schedule_service_integration(self): - """Test integration with ScheduleService patterns.""" + def test_schedule_policy_integration(self): + """Test visual scheduling policy output against the scheduler.""" from core.workflow.nodes.trigger_schedule.entities import VisualConfig - from services.trigger.schedule_service import ScheduleService + from services.trigger.schedule_policy import visual_to_cron # Test enhanced syntax through visual config conversion visual_configs = [ @@ -316,7 +316,7 @@ def test_schedule_service_integration(self): for test_case in visual_configs: with self.subTest(frequency=test_case["frequency"]): - cron_expr = ScheduleService.visual_to_cron(test_case["frequency"], test_case["config"]) + cron_expr = visual_to_cron(test_case["frequency"], test_case["config"]) assert cron_expr == test_case["expected_cron"] # Verify the generated cron expression is valid diff --git a/api/tests/unit_tests/models/test_agent.py b/api/tests/unit_tests/models/test_agent.py index 6dff7dcb7bf844..b93fb120bd3c32 100644 --- a/api/tests/unit_tests/models/test_agent.py +++ b/api/tests/unit_tests/models/test_agent.py @@ -6,6 +6,7 @@ import sqlalchemy as sa from sqlalchemy.exc import IntegrityError +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigDraft, @@ -24,7 +25,6 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig diff --git a/api/tests/unit_tests/models/test_model.py b/api/tests/unit_tests/models/test_model.py index fb7180a41ee3e8..b6dd5376333cff 100644 --- a/api/tests/unit_tests/models/test_model.py +++ b/api/tests/unit_tests/models/test_model.py @@ -593,58 +593,6 @@ def test_workflow_with_session_returns_none_when_unpublished(self, sqlite_sessio with Session(expire_on_commit=False) as unbound_session: assert app.workflow_with_session(session=unbound_session) is None - @pytest.mark.parametrize("sqlite_session", [(App, AppModelConfig)], indirect=True) - def test_deleted_tools_checks_plugin_builtin_providers_through_core_plugin_service( - self, sqlite_session: Session - ): - """Plugin-backed built-in tools are checked through core PluginService.""" - # Arrange - app = App( - tenant_id="tenant-1", - name="Test App", - mode=AppMode.CHAT, - enable_site=True, - enable_api=False, - created_by=str(uuid4()), - ) - sqlite_session.add(app) - sqlite_session.flush() - app_model_config = AppModelConfig( - app_id=app.id, - agent_mode=json.dumps( - { - "enabled": True, - "strategy": "function_call", - "tools": [ - { - "provider_type": "builtin", - "provider_id": "langgenius/openai/openai", - "tool_name": "chat", - "tool_parameters": {}, - } - ], - "prompt": None, - } - ), - ) - sqlite_session.add(app_model_config) - sqlite_session.flush() - app.app_model_config_id = app_model_config.id - sqlite_session.flush() - - # Act - with ( - patch("core.tools.tool_manager.ToolManager.get_hardcoded_provider", side_effect=Exception), - patch("core.plugin.plugin_service.PluginService.check_tools_existence", return_value=[False]) as exists, - ): - result = app.deleted_tools_with_session(session=sqlite_session) - - # Assert - assert result == [{"type": "builtin", "tool_name": "chat", "provider_id": "langgenius/openai/openai"}] - exists.assert_called_once() - assert exists.call_args.args[0] == "tenant-1" - assert [str(provider_id) for provider_id in exists.call_args.args[1]] == ["langgenius/openai/openai"] - class TestAppModelConfig: """Test suite for AppModelConfig model.""" diff --git a/api/tests/unit_tests/models/test_workflow.py b/api/tests/unit_tests/models/test_workflow.py index b3c2af1396b834..3e45870afd4284 100644 --- a/api/tests/unit_tests/models/test_workflow.py +++ b/api/tests/unit_tests/models/test_workflow.py @@ -13,12 +13,14 @@ from constants import HIDDEN_VALUE from core.helper import encrypter +from core.tools.entities.tool_entities import ToolProviderType from core.workflow.file_reference import build_file_reference from core.workflow.llm_environment_variable import LLMEnvironmentVariable from extensions.storage.storage_type import StorageType from factories import variable_factory from factories.variable_factory import build_segment from fields.workflow_run_fields import ( + WorkflowRunNodeExecutionResponse, node_execution_response_source, workflow_run_pagination_response_source, workflow_run_response_source, @@ -57,6 +59,8 @@ is_system_variable_editable, ) from models.workflow import WorkflowType as ModelWorkflowType +from services.tools.provider_queries import ToolProviderIcons +from services.tools.tool_manager import ToolManager class TestConversationVariableCases: @@ -1778,15 +1782,55 @@ def test_returns_none_when_role_is_account(self, sqlite_session: Session) -> Non assert execution.created_by_end_user(session=sqlite_session) is None class TestNodeExecutionResponseSource: + @pytest.mark.parametrize("node_type", [BuiltinNodeTypes.TOOL, BuiltinNodeTypes.DATASOURCE]) + def test_provider_icons_survive_response_serialization( + self, + sqlite_session: Session, + monkeypatch: pytest.MonkeyPatch, + node_type: str, + *, + tool_providers: ToolProviderIcons, + ) -> None: + execution = _execution(CreatorUserRole.ACCOUNT, created_by="account") + execution.id = "execution" + execution.node_type = node_type + execution.execution_metadata = json.dumps( + { + "tool_info": {"provider_type": "builtin", "provider_id": "provider"}, + "datasource_info": {"icon": "source-icon"}, + } + ) + + def tool_icon( + *, tenant_id: str, provider_type: ToolProviderType, provider_id: str, tool_providers: ToolProviderIcons + ) -> str: + assert tool_providers is not None + assert (tenant_id, provider_type, provider_id) == (execution.tenant_id, "builtin", "provider") + return "tool-icon" + + monkeypatch.setattr(ToolManager, "get_tool_icon", tool_icon) + response = WorkflowRunNodeExecutionResponse.model_validate( + node_execution_response_source( + execution, + session=sqlite_session, + tool_providers=tool_providers, + ) + ) + assert response.extras == {"icon": "tool-icon" if node_type == BuiltinNodeTypes.TOOL else "source-icon"} + def test_accessors_resolve_via_wrapped_session_and_other_attributes_proxy( - self, sqlite_session: Session + self, sqlite_session: Session, *, tool_providers: ToolProviderIcons ) -> None: account = Account(name="Test Account", email="test@example.com") sqlite_session.add(account) sqlite_session.flush() execution = _execution(CreatorUserRole.ACCOUNT, created_by=account.id) - source = node_execution_response_source(execution, session=sqlite_session) + source = node_execution_response_source( + execution, + session=sqlite_session, + tool_providers=tool_providers, + ) resolved = source.created_by_account assert resolved is not None diff --git a/api/tests/unit_tests/pyrefly.toml b/api/tests/unit_tests/pyrefly.toml index bdf577fc59823e..03e2d27ab849c9 100644 --- a/api/tests/unit_tests/pyrefly.toml +++ b/api/tests/unit_tests/pyrefly.toml @@ -29,7 +29,6 @@ project-excludes = [ "controllers/console/app/test_mcp_server.py", "controllers/console/app/test_message.py", "controllers/console/app/test_model_config.py", - "controllers/console/app/test_workflow.py", "controllers/console/app/test_workflow_node_output_inspector.py", "controllers/console/app/test_workflow_run.py", "controllers/console/app/test_wraps.py", @@ -48,7 +47,6 @@ project-excludes = [ "controllers/console/datasets/test_hit_testing.py", "controllers/console/datasets/test_hit_testing_base.py", "controllers/console/datasets/test_metadata.py", - "controllers/console/snippets/test_snippet_workflow.py", "controllers/console/tag/test_tags.py", "services/knowledge/test_dataset_read_service.py", "controllers/console/test_files.py", diff --git a/api/tests/unit_tests/repositories/test_agent_execution_repository.py b/api/tests/unit_tests/repositories/test_agent_execution_repository.py new file mode 100644 index 00000000000000..d6d6bde20b17bf --- /dev/null +++ b/api/tests/unit_tests/repositories/test_agent_execution_repository.py @@ -0,0 +1,149 @@ +"""Execution retention uses candidate-indexed references rather than graph scans.""" + +from datetime import timedelta +from functools import partial + +import pytest +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker + +from graphon.enums import WorkflowExecutionStatus +from libs.datetime_utils import naive_utc_now +from models.agent import WorkflowAgentNodeBinding +from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom +from models.workflow import WorkflowRun, WorkflowType +from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository +from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository +from tests.unit_tests.workflow_execution import debug_lease, execution_binding, set_debug_deadline + + +def _run(run_id: str, status: WorkflowExecutionStatus) -> WorkflowRun: + return WorkflowRun( + id=run_id, + tenant_id="tenant-1", + app_id="app-1", + workflow_id="workflow-1", + type=WorkflowType.WORKFLOW, + triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, + version="draft", + status=status, + created_by_role=CreatorUserRole.ACCOUNT, + created_by="account-1", + graph='{"note": "Retention must not select or decode this graph"}', + ) + + +def _reference(run_id: str, agent_id: str) -> WorkflowAgentNodeBinding: + return execution_binding(workflow_run_id=run_id, tenant_id="tenant-1", app_id="app-1", agent_id=agent_id) + + +def test_retention_queries_candidates_through_agent_index(sqlite_session_factory: sessionmaker[Session]) -> None: + with sqlite_session_factory.begin() as session: + for i in range(100): + run_id = f"unrelated-{i}" + session.add(_run(run_id, WorkflowExecutionStatus.RUNNING)) + session.flush() + session.add(_reference(run_id, f"other-agent-{i}")) + for run_id, status in ( + ("running", WorkflowExecutionStatus.RUNNING), + ("paused", WorkflowExecutionStatus.PAUSED), + ("ended", WorkflowExecutionStatus.STOPPED), + ): + session.add(_run(run_id, status)) + session.flush() + session.add(_reference(run_id, f"agent-{run_id}")) + statements: list[tuple[str, tuple[object, ...]]] = [] + engine = sqlite_session_factory.kw["bind"] + + def record(_conn: object, _cursor: object, sql: str, parameters: tuple[object, ...], *_args: object) -> None: + statements.append((sql, parameters)) + + event.listen(engine, "before_cursor_execute", record) + try: + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids( + session, "tenant-1", ["agent-running", "agent-paused", "agent-ended"] + ) == {"agent-running", "agent-paused"} + assert not session.identity_map # No WorkflowRun graph or ORM rows were materialized. + finally: + event.remove(engine, "before_cursor_execute", record) + assert len(statements) == 1 + sql, parameters = statements[0] + assert "graph" not in sql + with engine.connect() as connection: + plan = connection.exec_driver_sql("EXPLAIN QUERY PLAN " + sql, parameters).all() + assert any("SEARCH" in row[3] and "workflow_agent_node_binding_agent_idx" in row[3] for row in plan) + + +@pytest.mark.parametrize("field", ["tenant_id", "app_id", "workflow_id"]) +def test_retention_requires_complete_owner_chain(sqlite_session_factory: sessionmaker[Session], field: str) -> None: + with sqlite_session_factory.begin() as session: + session.add(_run("run", WorkflowExecutionStatus.PAUSED)) + session.flush() + reference = _reference("run", "agent-1") + setattr(reference, field, "other-owner") + session.add(reference) + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == set() + + +@pytest.mark.parametrize("cancel", [False, True]) +def test_execution_finish_keeps_retry_candidates_including_cancelled_reservation( + sqlite_session_factory: sessionmaker[Session], cancel: bool +) -> None: + with sqlite_session_factory.begin() as session: + session.add(_run("run", WorkflowExecutionStatus.RUNNING if cancel else WorkflowExecutionStatus.PAUSED)) + session.flush() + if cancel: + set_debug_deadline(session, "run", naive_utc_now() + timedelta(minutes=1)) + session.add(_reference("run", "agent-1")) + repository = WorkflowAgentExecutionRepository(sqlite_session_factory) + finish = partial( + WorkflowExecutionWriteRepository(sqlite_session_factory).finish, + tenant_id="tenant-1", + app_id="app-1", + workflow_id="workflow-1", + execution_id="run", + ) + if not cancel: + assert finish() == WorkflowExecutionStatus.PAUSED + assert ( + repository.finished_agent_ids( + tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", execution_id="run" + ) + == set() + ) + assert finish(status=WorkflowExecutionStatus.STOPPED, cancel=cancel) == WorkflowExecutionStatus.STOPPED + with sqlite_session_factory() as session: + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == set() + assert session.scalar(select(WorkflowAgentNodeBinding)) is not None + run = session.get(WorkflowRun, "run") + assert run is not None + assert run.status == WorkflowExecutionStatus.STOPPED + if cancel: + reservation = debug_lease(session, "run") + assert reservation is not None + assert reservation.expires_at is not None + assert reservation.expires_at <= naive_utc_now() + assert finish(cancel=cancel) == WorkflowExecutionStatus.STOPPED + assert repository.finished_agent_ids( + tenant_id="tenant-1", app_id="app-1", workflow_id="workflow-1", execution_id="run" + ) == {"agent-1"} + + +def test_execution_pins_are_not_reported_as_editable_workflow_nodes(sqlite_session: Session) -> None: + from services.agent.composer_service import AgentComposerService + + pin = _reference("run", "agent-1") + pin.current_snapshot_id = "snapshot-1" + draft = _reference("workflow-1", "agent-1") + draft.workflow_version = "draft" + draft.current_snapshot_id = "snapshot-1" + sqlite_session.add_all([pin, draft]) + sqlite_session.commit() + + impact = AgentComposerService.calculate_impact( + session=sqlite_session, tenant_id="tenant-1", current_snapshot_id="snapshot-1" + ) + assert impact["workflow_node_count"] == 1 + assert impact["bindings"] == [{"app_id": "app-1", "workflow_id": "workflow-1", "node_id": "agent-1"}] diff --git a/api/tests/unit_tests/repositories/test_app_definition_query_repository.py b/api/tests/unit_tests/repositories/test_app_definition_query_repository.py index f2e11d70a9fe8b..8e23a752b7891b 100644 --- a/api/tests/unit_tests/repositories/test_app_definition_query_repository.py +++ b/api/tests/unit_tests/repositories/test_app_definition_query_repository.py @@ -9,7 +9,6 @@ from sqlalchemy.orm import Session, SessionTransaction, sessionmaker from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from core.tools.entities.tool_entities import ApiProviderSchemaType from machinery.context import AppRequestContext from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus @@ -31,6 +30,7 @@ AppDefinitionQueryRepository, _get_public_agent_parameter_config, ) +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionSummary, AppParameterConfig, @@ -935,7 +935,7 @@ def test_published_agent_app_parameters_use_soul_file_upload( @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Session): +def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:unbound") app_model = _app_model(tenant_id=tenant_id) @@ -944,7 +944,7 @@ def test_published_agent_app_parameters_requires_bound_agent(sqlite_session: Ses @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_session: Session): +def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_session: Session) -> None: requested_tenant_id = _stable_uuid("tenant:requested") agent_id = _stable_uuid("agent:cross-tenant") app_model = _app_model(tenant_id=requested_tenant_id) @@ -965,7 +965,7 @@ def test_published_agent_app_parameters_requires_existing_active_agent(sqlite_se @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) def test_published_agent_app_parameters_requires_published_agent( active_config_is_published: bool, sqlite_session: Session -): +) -> None: tenant_id = _stable_uuid(f"tenant:published:{active_config_is_published}") agent_id = _stable_uuid(f"agent:published:{active_config_is_published}") app_model = _app_model(tenant_id=tenant_id) @@ -983,7 +983,9 @@ def test_published_agent_app_parameters_requires_published_agent( @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_allows_unpublished_draft_with_active_snapshot( + sqlite_session: Session, +) -> None: tenant_id = _stable_uuid("tenant:unpublished-draft") agent_id = _stable_uuid("agent:unpublished-draft") snapshot_id = _stable_uuid("snapshot:unpublished-draft") @@ -1014,7 +1016,7 @@ def test_published_agent_app_parameters_allows_unpublished_draft_with_active_sna @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:never-published") agent_id = _stable_uuid("agent:never-published") snapshot_id = _stable_uuid("snapshot:never-published") @@ -1041,7 +1043,7 @@ def test_published_agent_app_parameters_rejects_seeded_unpublished_snapshot(sqli @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_requires_published_snapshot(sqlite_session: Session): +def test_published_agent_app_parameters_requires_published_snapshot(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:missing-snapshot") agent_id = _stable_uuid("agent:missing-snapshot") app_model = _app_model(tenant_id=tenant_id) @@ -1065,7 +1067,7 @@ def test_published_agent_app_parameters_requires_published_snapshot(sqlite_sessi @pytest.mark.parametrize("sqlite_session", [(Agent, AgentConfigSnapshot, AgentConfigRevision)], indirect=True) -def test_published_agent_app_parameters_allows_missing_legacy_app_model_config(sqlite_session: Session): +def test_published_agent_app_parameters_allows_missing_legacy_app_model_config(sqlite_session: Session) -> None: tenant_id = _stable_uuid("tenant:no-legacy-config") agent_id = _stable_uuid("agent:no-legacy-config") snapshot_id = _stable_uuid("snapshot:no-legacy-config") diff --git a/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py b/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py index b9014ef32dc826..f018849bef407e 100644 --- a/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py +++ b/api/tests/unit_tests/repositories/test_sqlalchemy_execution_extra_content_repository.py @@ -11,7 +11,11 @@ from libs.datetime_utils import naive_utc_now from models.execution_extra_content import HumanInputContent as HumanInputContentModel from models.human_input import HumanInputForm, HumanInputFormRecipient -from models.human_input_entities import FormDefinition, ParagraphInputConfig, UserActionConfig +from models.human_input_entities import ( + FormDefinition, + ParagraphInputConfig, + UserActionConfig, +) from repositories.sqlalchemy_execution_extra_content_repository import SQLAlchemyExecutionExtraContentRepository diff --git a/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py b/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py index 1346134f015f61..770e085d7806d0 100644 --- a/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py +++ b/api/tests/unit_tests/repositories/test_workflow_app_log_query_repository.py @@ -16,8 +16,8 @@ from models.model import EndUser from models.trigger import WorkflowTriggerLog from models.workflow import WorkflowAppLog, WorkflowAppLogCreatedFrom, WorkflowRun, WorkflowType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import WorkflowAppLogAccount, WorkflowAppLogEndUser +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import WorkflowAppLogAccount, WorkflowAppLogEndUser def _log( @@ -117,7 +117,7 @@ def test_get_paginated_returns_detached_actor_records_by_role_when_ids_overlap( sqlite_session.commit() sqlite_session.close() - result = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory).get_paginated( + result = WorkflowAppLogRepository(session_factory=sqlite_session_factory).get_paginated( tenant_id=tenant.id, app_id="app-1", ) @@ -143,7 +143,7 @@ def test_get_paginated_returns_detached_actor_records_by_role_when_ids_overlap( def test_get_paginated_preserves_missing_account_filter_error( sqlite_session_factory: sessionmaker[Session], ) -> None: - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) with pytest.raises(ValueError, match=r"^Account not found: missing@example\.com$"): repository.get_paginated( @@ -155,7 +155,7 @@ def test_get_paginated_preserves_missing_account_filter_error( def test_enum_value_rejects_missing_required_value() -> None: with pytest.raises(ValueError, match="Required enum value is missing"): - WorkflowAppLogQueryRepository._enum_value(None) + WorkflowAppLogRepository._enum_value(None) def test_get_paginated_projects_plugin_workflow_run_summary( @@ -172,7 +172,7 @@ def test_get_paginated_projects_plugin_workflow_run_summary( sqlite_session.add_all([run, log]) sqlite_session.commit() - result = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory).get_paginated( + result = WorkflowAppLogRepository(session_factory=sqlite_session_factory).get_paginated( tenant_id="tenant-1", app_id="app-1", ) @@ -200,7 +200,7 @@ def test_get_paginated_keeps_log_when_workflow_run_is_missing( sqlite_session.add(log) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) result = repository.get_paginated(tenant_id="tenant-1", app_id="app-1") assert result.total == 1 @@ -253,7 +253,7 @@ def test_get_paginated_includes_trigger_metadata_only_with_detail( ) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) assert repository.get_paginated(tenant_id="tenant-1", app_id="app-1").data[0].details is None assert repository.get_paginated( @@ -290,7 +290,7 @@ def test_get_paginated_does_not_attach_workflow_run_from_another_scope( ) sqlite_session.commit() - repository = WorkflowAppLogQueryRepository(session_factory=sqlite_session_factory) + repository = WorkflowAppLogRepository(session_factory=sqlite_session_factory) result = repository.get_paginated(tenant_id="tenant-1", app_id="app-1") assert result.total == 1 diff --git a/api/tests/unit_tests/repositories/test_workflow_tool_source_repository.py b/api/tests/unit_tests/repositories/test_workflow_tool_source_repository.py index 12be6dd1ce7c77..8e72c83b84c480 100644 --- a/api/tests/unit_tests/repositories/test_workflow_tool_source_repository.py +++ b/api/tests/unit_tests/repositories/test_workflow_tool_source_repository.py @@ -2,20 +2,47 @@ from functools import partial import pytest -from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.orm import Session, object_session, sessionmaker from core.tools.workflow_as_tool.repository import WorkflowToolSource from core.workflow.workflow_tool_container_types import WorkflowToolContainerPayload +from extensions.application_services.workflow import WorkflowExecutionDependencies from models.model import App, AppMode from models.tools import WorkflowToolProvider from models.workflow import Workflow, WorkflowKind, WorkflowType from repositories.workflow_tool_source_repository import SQLAlchemyWorkflowToolSourceRepository +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow_tool_source_service import WorkflowToolSourceService + + +def _repository( + sessions: sessionmaker[Session], + workflow_runtime: WorkflowRuntime | None, +) -> SQLAlchemyWorkflowToolSourceRepository: + return SQLAlchemyWorkflowToolSourceRepository( + sessions, + workflow_runtime=workflow_runtime, + draft_variable_saver=None, + ) + + +def _source_service( + sessions: sessionmaker[Session], + workflow_runtime: WorkflowRuntime, +) -> WorkflowToolSourceService: + repository = _repository(sessions, workflow_runtime) + return WorkflowToolSourceService( + repository, + workflow_runtime=workflow_runtime, + draft_variable_saver=None, + ) def test_materialized_tool_sources_keep_the_published_version_and_tenant( sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, ) -> None: - from services.workflow_tool_source_service import WorkflowToolSourceService + from services.workflow_tool_source_service import WorkflowToolSnapshot _save_source(sqlite_session_factory) with sqlite_session_factory.begin() as session: @@ -33,7 +60,7 @@ def test_materialized_tool_sources_keep_the_published_version_and_tenant( session.flush() provider_id = provider.id graph = {"nodes": [{"data": {"type": "tool", "provider_type": "workflow", "provider_id": provider_id}}]} - service = WorkflowToolSourceService(SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory)) + service = _source_service(sqlite_session_factory, workflow_runtime) sources = service.load(tenant_id=_TENANT_ID, graph_config=graph).sources assert set(sources) == {_PUBLISHED_WORKFLOW_ID} @@ -56,14 +83,23 @@ def test_materialized_tool_sources_keep_the_published_version_and_tenant( resumed = service.load(tenant_id=_TENANT_ID, graph_config=graph, suspended_tools=[suspended]).sources assert set(resumed) == {_PUBLISHED_WORKFLOW_ID, _LATEST_WORKFLOW_ID} assert resumed[_PUBLISHED_WORKFLOW_ID].graph_config == _GRAPH + pinned = service.load( + tenant_id=_TENANT_ID, + graph_config=graph, + suspended_tools=[suspended], + snapshot=WorkflowToolSnapshot(tools={}), + ) + assert set(pinned.sources) == {_PUBLISHED_WORKFLOW_ID} + assert pinned.sources[_PUBLISHED_WORKFLOW_ID].graph_config == _GRAPH with pytest.raises(ValueError, match="source was not found"): service.load(tenant_id=_OTHER_TENANT_ID, graph_config=graph, suspended_tools=[suspended]) def test_serialized_tool_snapshot_preserves_definitions_without_current_providers( sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, ) -> None: - from services.workflow_tool_source_service import WorkflowToolSnapshot, WorkflowToolSourceService + from services.workflow_tool_source_service import WorkflowToolSnapshot _save_source(sqlite_session_factory) variables = [ @@ -96,7 +132,7 @@ def test_serialized_tool_snapshot_preserves_definitions_without_current_provider session.flush() provider_id = provider.id graph = {"nodes": [{"data": {"type": "tool", "provider_type": "workflow", "provider_id": provider_id}}]} - service = WorkflowToolSourceService(SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory)) + service = _source_service(sqlite_session_factory, workflow_runtime) original = service.load(tenant_id=_TENANT_ID, graph_config=graph) snapshot = WorkflowToolSnapshot.model_validate_json(original.snapshot().model_dump_json()) with sqlite_session_factory.begin() as session: @@ -109,7 +145,9 @@ def test_serialized_tool_snapshot_preserves_definitions_without_current_provider latest = service.load(tenant_id=_TENANT_ID, graph_config=graph) assert latest.tools[provider_id].workflow_id == _LATEST_WORKFLOW_ID assert latest.tools[provider_id].entity.parameters == [] - assert service.load(tenant_id=_TENANT_ID, graph_config=graph, snapshot=WorkflowToolSnapshot(tools={})).tools == {} + empty = service.load(tenant_id=_TENANT_ID, graph_config=graph, snapshot=WorkflowToolSnapshot(tools={})) + assert empty.tools == {} + assert empty.sources == {} for remove_provider in (False, True): if remove_provider: @@ -141,13 +179,13 @@ def test_serialized_tool_snapshot_preserves_definitions_without_current_provider def test_saved_provider_aliases_keep_distinct_definitions_for_the_same_source( sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, ) -> None: from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolEntity, ToolIdentity from services.workflow_tool_source_service import ( WorkflowToolDefinition, WorkflowToolSnapshot, - WorkflowToolSourceService, ) _save_source(sqlite_session_factory) @@ -170,7 +208,7 @@ def test_saved_provider_aliases_keep_distinct_definitions_for_the_same_source( for provider_id in ("first-provider", "second-provider") } ) - restored = WorkflowToolSourceService(SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory)).load( + restored = _source_service(sqlite_session_factory, workflow_runtime).load( tenant_id=_TENANT_ID, graph_config={}, snapshot=WorkflowToolSnapshot.model_validate_json(snapshot.model_dump_json()), @@ -245,11 +283,51 @@ def _save_source(session_factory: sessionmaker[Session]) -> None: session.add_all((app, *workflows)) +def test_provider_materialization_happens_after_repository_session_closes( + sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from services.tools.workflow.tool import WorkflowTool + + _save_source(sqlite_session_factory) + with sqlite_session_factory.begin() as session: + provider = WorkflowToolProvider( + tenant_id=_TENANT_ID, + app_id=_APP_ID, + version=_PUBLISHED_VERSION, + name="source", + label="Source", + icon="{}", + user_id=_ACCOUNT_ID, + description="Source", + ) + session.add(provider) + session.flush() + provider_id = provider.id + + repository = _repository(sqlite_session_factory, workflow_runtime) + build_tool = repository._build_tool + materialization_sessions: list[Session | None] = [] + + def build_detached_tool(provider: WorkflowToolProvider) -> WorkflowTool: + materialization_sessions.append(object_session(provider)) + return build_tool(provider) + + monkeypatch.setattr(repository, "_build_tool", build_detached_tool) + + sources = repository.get_by_provider_ids(tenant_id=_TENANT_ID, provider_ids=(provider_id,)) + + assert materialization_sessions == [None] + assert len(sources) == 1 + assert isinstance(sources[0].tool, WorkflowTool) + + def test_get_source_returns_requested_workflow_after_a_new_version_is_published( sqlite_session_factory: sessionmaker[Session], ) -> None: _save_source(sqlite_session_factory) - repository = SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory) + repository = _repository(sqlite_session_factory, None) source = repository.get_source( tenant_id=_TENANT_ID, @@ -283,7 +361,7 @@ def test_get_source_rejects_unavailable_source( version: str, ) -> None: _save_source(sqlite_session_factory) - repository = SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory) + repository = _repository(sqlite_session_factory, None) assert ( repository.get_source( @@ -309,7 +387,7 @@ def test_get_source_returns_workflow_kind_enum( assert workflow is not None workflow.kind = None if stored_kind is None else WorkflowKind.value_of(stored_kind) - source = SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory).get_source( + source = _repository(sqlite_session_factory, None).get_source( tenant_id=_TENANT_ID, app_id=_APP_ID, workflow_id=_PUBLISHED_WORKFLOW_ID, @@ -323,9 +401,9 @@ def test_get_source_returns_workflow_kind_enum( @pytest.mark.parametrize("allow_human_input", [False, True]) def test_materialize_recursive_tools_checks_nested_human_input( sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, allow_human_input: bool, ) -> None: - from services.workflow_tool_source_service import WorkflowToolSourceService _save_source(sqlite_session_factory) with sqlite_session_factory.begin() as session: @@ -370,7 +448,7 @@ def graph(provider_id: str) -> dict[str, list[dict[str, object]]]: child.app_id = child_app_id parent.graph = json.dumps(graph(child_id)) child.graph = json.dumps({"nodes": [*graph(parent_id)["nodes"], {"data": {"type": "human-input"}}]}) - service = WorkflowToolSourceService(SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory)) + service = _source_service(sqlite_session_factory, workflow_runtime) if allow_human_input: materialized = service.load(tenant_id=_TENANT_ID, graph_config=graph(parent_id), allow_human_input=True) assert set(materialized.sources) == {_PUBLISHED_WORKFLOW_ID, _LATEST_WORKFLOW_ID} @@ -389,17 +467,18 @@ def graph(provider_id: str) -> dict[str, list[dict[str, object]]]: @pytest.mark.parametrize("resuming", [False, True]) def test_running_tool_keeps_materialized_definition_after_republication( sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, resuming: bool, ) -> None: - from core.workflow.node_factory import DifyNodeFactory - from core.workflow.workflow_tool_container_handler import WorkflowToolContainerHandler from graphon.engine import Engine from graphon.engine_events import NodeRunSucceededEvent from graphon.engine_events.graph import GraphRunSucceededEvent from graphon.graph import Graph from graphon.nodes.tool.entities import ToolNodeData from graphon.runtime import RuntimeState, VariablePool - from services.workflow_tool_source_service import WorkflowToolSourceService + from services.tools.workflow.tool import WorkflowTool + from services.workflow.execution.adapters.node_factory import DifyNodeFactory + from services.workflow.execution.adapters.workflow_tool_container_handler import WorkflowToolContainerHandler from tests.workflow_test_utils import build_test_graph_init_params _save_source(sqlite_session_factory) @@ -446,33 +525,31 @@ def test_running_tool_keeps_materialized_definition_after_republication( ], "edges": [{"source": "start", "target": "loop"}, {"source": "loop-start", "target": "tool"}], } - sources = ( - WorkflowToolSourceService(SQLAlchemyWorkflowToolSourceRepository(sqlite_session_factory)) - .load( - tenant_id=_TENANT_ID, - graph_config=graph, - suspended_tools=[ - WorkflowToolContainerPayload( - source_app_id=_APP_ID, - source_workflow_id=_PUBLISHED_WORKFLOW_ID, - source_workflow_version=_PUBLISHED_VERSION, - call_depth=1, - ) - ] - if resuming - else (), - ) - .sources + materialized = _source_service(sqlite_session_factory, workflow_runtime).load( + tenant_id=_TENANT_ID, + graph_config=graph, + suspended_tools=[ + WorkflowToolContainerPayload( + source_app_id=_APP_ID, + source_workflow_id=_PUBLISHED_WORKFLOW_ID, + source_workflow_version=_PUBLISHED_VERSION, + call_depth=1, + ) + ] + if resuming + else (), ) + sources = materialized.sources source = sources[_PUBLISHED_WORKFLOW_ID] - assert source.tool is not None + assert isinstance(source.tool, WorkflowTool) state = RuntimeState(workflow_id="caller", variable_pool=VariablePool(), start_at=1) factory = DifyNodeFactory( init_params=build_test_graph_init_params( workflow_id="caller", graph_config=graph, tenant_id=_TENANT_ID, app_id=_APP_ID ), runtime_state=state, - workflow_tools={provider_id: source.tool}, + workflow_runtime=workflow_runtime, + workflow_tools=materialized.tools, ) completed_nodes = [] engine = Engine( diff --git a/api/tests/unit_tests/repositories/workflow/test_definition_repository.py b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py new file mode 100644 index 00000000000000..24fc69413e47b2 --- /dev/null +++ b/api/tests/unit_tests/repositories/workflow/test_definition_repository.py @@ -0,0 +1,2038 @@ +"""Persistence boundaries use real short SQLite sessions, including rollback.""" + +import json +from collections.abc import Callable +from dataclasses import replace +from typing import cast +from unittest.mock import Mock, create_autospec, patch +from uuid import uuid4 + +import pytest +from sqlalchemy import Select, event, func, select +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +from core.workflow.llm_environment_variable import LLMEnvironmentVariable +from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_workflow_definition_gateway +from extensions.application_services.workflow_variables import ( + build_console_workflow_variables, + build_workflow_variable_service, +) +from fields.workflow_fields import WorkflowResponse +from graphon.model_runtime.entities.model_entities import ModelType +from graphon.variables import StringSegment, StringVariable +from machinery.context import RequestContext +from models import Account, App +from models.account import TenantAccountJoin, TenantAccountRole +from models.agent import WorkflowAgentNodeBinding +from models.agent_config_entities import WorkflowNodeJobConfig +from models.dataset import AppDatasetJoin +from models.enums import CredentialSourceType +from models.model import AppMode +from models.provider import LoadBalancingModelConfig +from models.tools import BuiltinToolProvider +from models.workflow import Workflow, WorkflowDraftVariable, WorkflowVersionCounter +from repositories.app.console_repository import ConsoleAppRepository +from repositories.credentials.query_repository import CredentialQueryRepository +from repositories.workflow.definition_repository import ( + WorkflowDefinitionRepository, + WorkflowDefinitionStore, + workflow_snapshot, +) +from repositories.workflow.draft_repository import WorkflowDraftRepository +from services import workflow_service as workflow_module +from services.agent.workflow_contracts import AgentSkillReader, WorkflowAgentBindingStore +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.app.console_service import ConsoleAppNotFoundError +from services.data_migration.entities import ImportTarget +from services.data_migration.import_service import MigrationImportService +from services.errors.app import TriggerNodeLimitExceededError, WorkflowHashNotEqualError +from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.tools.workflow_tools_manage_service import WorkflowToolManageService +from services.workflow import definition_gateway as gateway_module +from services.workflow.console_service import ( + ConsoleWorkflowService, + WorkflowAccess, + WorkflowAppLookup, + WorkflowConversion, + WorkflowPresence, + WorkflowRuntime, +) +from services.workflow.contracts import ( + DraftSyncCommand, + PreparedDraftSync, + ValidatedWorkflowPublication, + WorkflowOwner, + WorkflowPublication, + WorkflowSnapshot, +) +from services.workflow.definition_gateway import WorkflowDefinitionGateway +from services.workflow.draft_service import DraftAgentRetirement, WorkflowDraftService +from services.workflow.environment_variable_service import prepare_environment_variables +from services.workflow_service import WorkflowService +from tests.helpers import billing_builders as builders +from tests.helpers.unit.model_factories import make_account, make_app, make_tenant, make_workflow + +CONTEXT = RequestContext("request", None, "account-1", "tenant-1") + + +@pytest.mark.parametrize("kind", ["tool", "agent", "llm"]) +@pytest.mark.parametrize("denied", [False, True]) +def test_publication_credential_policy_runs_after_read_transactions_close( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, kind: str, denied: bool +) -> None: + provider = "langgenius/openai/openai" + tool = BuiltinToolProvider( + tenant_id="tenant-1", user_id="account-1", provider=provider, name="default", is_default=True + ) + data: dict[str, object] = {"type": kind, "provider_id": provider} + if kind == "agent": + data["agent_parameters"] = {"tools": {"value": [{"provider_name": provider}]}} + if kind == "llm": + data["model"] = {"provider": provider, "name": "model"} + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps({"nodes": [{"id": "node", "data": data}], "edges": []}) + session.add(tool) + for tenant, credential, source in [ + ("tenant-1", "provider-credential", CredentialSourceType.PROVIDER), + ("tenant-1", "model-credential", CredentialSourceType.CUSTOM_MODEL), + ("foreign-tenant", "foreign-credential", CredentialSourceType.PROVIDER), + ]: + session.add( + LoadBalancingModelConfig( + tenant_id=tenant, + provider_name=provider, + model_name="model", + model_type=ModelType.LLM, + name=credential, + credential_id=credential, + credential_source_type=source, + ) + ) + sessions: list[Session] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + checked: list[str] = [] + + def check(credential_id: str, *_args: object, **_kwargs: object) -> None: + assert sessions + assert all(not session.in_transaction() for session in sessions) + checked.append(credential_id) + if denied: + raise ValueError("credential policy denied") + + workflows = WorkflowService(session_maker=seeded) + monkeypatch.setattr(workflow_module.SystemFeatureService, "is_plugin_manager_enabled", lambda: True) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + monkeypatch.setattr(workflows, "_validate_llm_model_config", lambda *_args: None) + monkeypatch.setattr(workflows, "_is_load_balancing_enabled", lambda *_args: True) + monkeypatch.setattr("core.helper.credential_utils.check_credential_policy_compliance", check) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=workflows, + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + event.listen(seeded, "after_begin", opened) + try: + if denied: + with pytest.raises(ValueError, match="credential policy denied"): + lifecycle.validate_publish(CONTEXT, "app-1") + else: + assert lifecycle.validate_publish(CONTEXT, "app-1").workflow.id == "draft" + expected = {"provider-credential", "model-credential"} if kind == "llm" else {tool.id} + assert set(checked) == expected + assert checked + assert all(not session.in_transaction() for session in sessions) + finally: + event.remove(seeded, "after_begin", opened) + + +@pytest.fixture +def seeded(sqlite_session_factory: sessionmaker[Session]) -> sessionmaker[Session]: + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_account(), + make_tenant(), + TenantAccountJoin(tenant_id="tenant-1", account_id="account-1", role=TenantAccountRole.OWNER), + make_app(workflow_id="published", mode=AppMode.WORKFLOW, created_by="account-1"), + make_workflow(workflow_id="draft"), + make_workflow(workflow_id="published", version="1", version_number=1), + WorkflowVersionCounter(app_id="app-1", last_version_number=1), + ] + ) + return sqlite_session_factory + + +@pytest.fixture +def repository(seeded: sessionmaker[Session]) -> WorkflowDefinitionRepository: + return WorkflowDefinitionRepository(session_factory=seeded) + + +def test_published_result_is_complete_after_session_closes(repository: WorkflowDefinitionRepository) -> None: + record = repository.published(CONTEXT, "app-1") + assert record is not None + response = WorkflowResponse.model_validate(record, from_attributes=True).model_dump(mode="json") + assert response["id"] == "published" + assert response["version_number"] == 1 + assert response["created_by"]["id"] == "account-1" + assert response["tool_published"] is False + + +def test_update_version_remarks_masks_secrets_without_key_io( + seeded: sessionmaker[Session], repository: WorkflowDefinitionRepository, monkeypatch: pytest.MonkeyPatch +) -> None: + from core.helper.encrypter import full_mask_token + + environment = json.dumps( + { + "API_KEY": {"id": "secret-id", "name": "API_KEY", "value_type": "secret", "value": "ciphertext"}, + "COUNT": {"id": "number-id", "name": "COUNT", "value_type": "number", "value": 3}, + } + ) + with seeded.begin() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + workflow._environment_variables = environment + sessions: list[Session] = [] + + def opened(session: Session, *_args: object) -> None: + sessions.append(session) + + def key_io(**_kwargs: object) -> str: + assert all(not session.in_transaction() for session in sessions) + pytest.fail("A display-only projection must not wrap or unwrap secret values") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", key_io) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", key_io) + event.listen(seeded, "after_begin", opened) + try: + record = repository.update(CONTEXT, "app-1", "published", {"marked_comment": "New remarks"}) + assert record is not None + assert sessions + assert all(not session.in_transaction() for session in sessions) + assert {variable.name: variable.value for variable in record.environment_variables} == { + "API_KEY": full_mask_token(), + "COUNT": 3, + } + response = WorkflowResponse.model_validate(record, from_attributes=True) + assert response.marked_comment == "New remarks" + assert response.environment_variables[0].value == full_mask_token() + finally: + event.remove(seeded, "after_begin", opened) + with seeded() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + assert workflow.marked_comment == "New remarks" + assert workflow.updated_by == CONTEXT.account_id + assert workflow._environment_variables == environment + + +def test_version_counter_allocation_is_rolled_back_with_publication(seeded: sessionmaker[Session]) -> None: + from repositories.workflow.version_repository import allocate_version_number + + with seeded.begin() as session: + assert allocate_version_number(session=session, app_id="app-1") == 2 + with seeded() as session, session.begin(): + assert allocate_version_number(session=session, app_id="app-1") == 3 + session.rollback() + with seeded.begin() as session: + assert allocate_version_number(session=session, app_id="app-1") == 3 + + +@pytest.mark.parametrize("concurrent_edit", [False, True]) +def test_publication_copies_secret_ciphertext_without_key_io_under_locks( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: bool +) -> None: + ciphertext = json.dumps( + { + "secret-id": { + "id": "secret-id", + "name": "API_KEY", + "value_type": "secret", + "value": "wrapped-secret", + } + } + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = ciphertext + sessions: list[Session] = [] + unwrapped: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def unwrap(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + unwrapped.append(token) + if concurrent_edit: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = ciphertext.replace("wrapped-secret", "changed-secret") + return "plaintext" + + def wrap(**_kwargs: object) -> str: + pytest.fail("Publishing within the same tenant must preserve the encrypted value") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", unwrap) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", wrap) + build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + event.listen(seeded, "after_begin", opened) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish(CONTEXT, "app-1", marked_name="Secrets", marked_comment="") + else: + service.publish(CONTEXT, "app-1", marked_name="Secrets", marked_comment="") + finally: + event.remove(seeded, "after_begin", opened) + assert unwrapped == ["wrapped-secret"] + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + if concurrent_edit: + assert app.workflow_id == "published" + else: + assert app.workflow_id != "published" + published = session.get(Workflow, app.workflow_id) + assert published is not None + assert published._environment_variables == ciphertext + + +@pytest.mark.parametrize("entry", ["sync", "collaborative", "replace", "patch"]) +@pytest.mark.parametrize("concurrent_edit", [False, True]) +def test_draft_secret_preparation_releases_sessions_and_rechecks_snapshot( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, entry: str, concurrent_edit: bool +) -> None: + from constants import HIDDEN_VALUE + from graphon.variables import SecretVariable + + original = json.dumps( + {"API_KEY": {"id": "secret-id", "name": "API_KEY", "value_type": "secret", "value": "wrapped"}} + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft._environment_variables = original + source = workflow_snapshot(draft) + + sessions: list[Session] = [] + operations: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def decrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert token == "wrapped" + assert all(not session.in_transaction() for session in sessions) + operations.append("decrypt") + return "old-secret" + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + operations.append("encrypt") + if concurrent_edit and operations.count("encrypt") == 1: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": [], "concurrent": true}' + return "encrypted:" + token + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + console_variables = build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda *_args, **_kwargs: None) + monkeypatch.setattr(workflows, "validate_features_structure", lambda *_args: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + variables = [ + SecretVariable(id="secret-id", name="RENAMED_KEY", value=HIDDEN_VALUE), + SecretVariable(id="new-secret", name="NEW_KEY", value="new-secret"), + ] + + def save() -> None: + if entry in ("replace", "patch"): + console_variables.update_environment( + CONTEXT, + "app-1", + [variable.model_dump(mode="json") for variable in variables], + deleted_ids=[] if entry == "patch" else None, + ) + else: + mappings = [variable.model_dump(mode="json") for variable in variables] + service.sync( + CONTEXT, + "app-1", + DraftSyncCommand( + graph={"nodes": [], "edges": [], "saved": True}, + features={}, + unique_hash=source.hash, + is_collaborative=entry == "collaborative", + conversation_variables=[], + environment_variables=mappings if entry == "sync" else None, + environment_upserts=mappings if entry == "collaborative" else None, + environment_deletions=[], + ), + ) + + event.listen(seeded, "after_begin", opened) + try: + if concurrent_edit and entry in ("sync", "collaborative"): + with pytest.raises(WorkflowHashNotEqualError): + save() + else: + save() + finally: + event.remove(seeded, "after_begin", opened) + assert "decrypt" in operations + assert "encrypt" in operations + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if concurrent_edit and entry in ("sync", "collaborative"): + assert draft._environment_variables == original + else: + values = json.loads(draft._environment_variables) + assert values["RENAMED_KEY"]["value"] == "encrypted:old-secret" + assert values["NEW_KEY"]["value"] == "encrypted:new-secret" + if concurrent_edit: + assert draft.graph_dict["concurrent"] is True + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_dsl_secret_preparation_precedes_the_atomic_import_transaction( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rollback: bool +) -> None: + import yaml + + from constants.dsl_version import CURRENT_APP_DSL_VERSION + from extensions.application_services.workflow import build_app_dsl_service + from services.entities.dsl_entities import ImportStatus + + sessions: list[Session] = [] + encrypted: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == "tenant-1" + assert all(not session.in_transaction() for session in sessions) + encrypted.append(token) + return "encrypted:" + token + + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + monkeypatch.setattr(WorkflowService, "validate_graph_structure", lambda *_args, **_kwargs: None) + monkeypatch.setattr(WorkflowService, "validate_features_structure", lambda *_args, **_kwargs: None) + event.listen(Session, "after_begin", opened) + try: + with seeded() as session: + result = build_app_dsl_service(session).import_app( + account=make_account(tenant=make_tenant()), + import_mode="yaml-content", + app_id="app-1", + yaml_content=yaml.safe_dump( + { + "version": CURRENT_APP_DSL_VERSION, + "kind": "app", + "app": {"mode": "workflow", "name": "Imported"}, + "workflow": { + "graph": {"nodes": [], "edges": []}, + "features": {}, + "environment_variables": [ + {"id": "secret-id", "name": "KEY", "value_type": "secret", "value": "secret"} + ], + }, + } + ), + ) + assert result.status == ImportStatus.COMPLETED, result.error + if rollback: + session.rollback() + else: + session.commit() + finally: + event.remove(Session, "after_begin", opened) + assert encrypted == ["secret"] + with seeded() as session: + draft = session.get(Workflow, "draft") + app = session.get(App, "app-1") + assert draft is not None + assert app is not None + if rollback: + assert json.loads(draft._environment_variables) == {} + assert app.name != "Imported" + else: + assert json.loads(draft._environment_variables)["KEY"]["value"] == "encrypted:secret" + assert app.name == "Imported" + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_workflow_and_bindings_delete_atomically_before_agent_retirement( + seeded: sessionmaker[Session], rollback: bool +) -> None: + with seeded.begin() as session: + session.add(make_workflow(workflow_id="obsolete", version="old-version")) + session.add_all( + WorkflowAgentNodeBinding( + id=f"binding-{node}", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="obsolete", + workflow_version="old-version", + node_id=node, + binding_type=kind, + agent_id=agent, + node_job_config={}, + ) + for node, kind, agent in [ + ("one", WorkflowAgentBindingType.INLINE_AGENT, "inline-agent"), + ("two", WorkflowAgentBindingType.INLINE_AGENT, "inline-agent"), + ("roster", WorkflowAgentBindingType.ROSTER_AGENT, "roster-agent"), + ] + ) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway( + seeded, definitions, WorkflowService(seeded), drafts=WorkflowDraftRepository(seeded) + ) + ) + retired: list[list[str]] = [] + + def retire(context: RequestContext, agent_ids: list[str]) -> None: + assert context is CONTEXT + with seeded() as session: + assert session.get(Workflow, "obsolete") is None + assert session.scalar(select(func.count()).select_from(WorkflowAgentNodeBinding)) == 0 + retired.append(agent_ids) + + cast(Mock, service._runtime).retire_agents.side_effect = retire + + def fail_commit(_session: Session) -> None: + if rollback: + raise RuntimeError("delete commit failed") + + event.listen(seeded, "before_commit", fail_commit) + try: + if rollback: + with pytest.raises(RuntimeError, match="delete commit failed"): + service.delete(CONTEXT, WorkflowOwner("app-1"), "obsolete") + else: + service.delete(CONTEXT, WorkflowOwner("app-1"), "obsolete") + finally: + event.remove(seeded, "before_commit", fail_commit) + with seeded() as session: + assert (session.get(Workflow, "obsolete") is not None) is rollback + assert session.scalar(select(func.count()).select_from(WorkflowAgentNodeBinding)) == (3 if rollback else 0) + assert retired == ([] if rollback else [["inline-agent"]]) + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_dsl_replacement_and_variable_invalidation_are_atomic(seeded: sessionmaker[Session], rollback: bool) -> None: + variables = [ + WorkflowDraftVariable.new_node_variable( + app_id=app_id, + user_id=user_id, + node_id="node", + name="value", + value=StringSegment(value="old"), + node_execution_id="execution", + ) + for app_id, user_id in [("app-1", "user-1"), ("app-1", "user-2"), ("other-app", "user-1")] + ] + with seeded.begin() as session: + session.add_all(variables) + + def import_draft() -> None: + with seeded.begin() as session: + app = session.get(App, "app-1") + draft = session.get(Workflow, "draft") + account = session.get(Account, "account-1") + assert app is not None + assert draft is not None + assert account is not None + WorkflowDefinitionStore.sync_draft_workflow( + app_model=app, + session=session, + graph={"nodes": [], "edges": [], "imported": True}, + features={}, + prepared=PreparedDraftSync(workflow_snapshot(draft), None), + account_id=account.id, + conversation_variables=[], + clear_debug_variables=True, + ) + if rollback: + raise RuntimeError("import failed") + + if rollback: + with pytest.raises(RuntimeError, match="import failed"): + import_draft() + else: + import_draft() + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.graph_dict.get("imported", False) is not rollback + remaining = set(session.scalars(select(WorkflowDraftVariable.id))) + assert remaining == ({variable.id for variable in variables} if rollback else {variables[-1].id}) + + +def test_draft_projection_does_not_mutate_persisted_graph( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + WorkflowAgentPublishService, + "project_draft_bindings_to_graph", + lambda _self, **_kwargs: {"nodes": [{"id": "projected"}]}, + ) + repository = WorkflowDefinitionRepository(session_factory=seeded) + with repository.draft(CONTEXT, "app-1") as draft: + assert draft is not None + projected = WorkflowAgentPublishService(repository=draft.bindings).project_draft_bindings_to_graph( + draft_workflow=draft.workflow + ) + assert projected == {"nodes": [{"id": "projected"}]} + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.graph_dict == {"nodes": [], "edges": []} + + +def test_queries_and_updates_keep_complete_owner_chain( + repository: WorkflowDefinitionRepository, seeded: sessionmaker[Session] +) -> None: + with pytest.raises(ConsoleAppNotFoundError): + repository.published(CONTEXT._replace(active_workspace_id="tenant-2"), "app-1") + with seeded.begin() as session: + session.add(make_workflow(workflow_id="foreign", tenant_id="tenant-2", app_id="other-app", version="1")) + assert repository.update(CONTEXT, "app-1", "foreign", {"marked_name": "modified"}) is None + with seeded() as session: + foreign = session.get(Workflow, "foreign") + assert foreign is not None + assert foreign.marked_name == "" + + +def application(lifecycle: WorkflowDefinitionGateway) -> ConsoleWorkflowService[WorkflowAgentBindingStore]: + return ConsoleWorkflowService( + conversion=create_autospec(WorkflowConversion, instance=True), + agent_services=WorkflowAgentPublishService, + definitions=cast(WorkflowDefinitionRepository, lifecycle._definitions), + drafts=WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=cast(WorkflowDraftRepository, lifecycle._drafts), + lifecycle=lifecycle, + retirement=create_autospec(DraftAgentRetirement, instance=True), + ), + lifecycle=lifecycle, + runtime=create_autospec(WorkflowRuntime, instance=True), + apps=create_autospec(WorkflowAppLookup, instance=True), + presence=create_autospec(WorkflowPresence, instance=True), + access=create_autospec(WorkflowAccess, instance=True), + ) + + +def publish(lifecycle: WorkflowDefinitionGateway, snapshot: WorkflowSnapshot) -> WorkflowPublication: + # Supply the previously validated snapshot to exercise a concurrent edit or + # failed commit through the same orchestration as the console and migration. + with patch.object(lifecycle, "validate_publish", return_value=ValidatedWorkflowPublication(snapshot, ())): + result, _ = application(lifecycle).publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + return result + + +@pytest.mark.parametrize("fail_commit", [False, True]) +def test_publication_and_app_pointer_commit_atomically(seeded: sessionmaker[Session], fail_commit: bool) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + + def fail(_session: object) -> None: + raise RuntimeError("commit failed") + + if fail_commit: + event.listen(seeded, "before_commit", fail) + try: + if fail_commit: + with pytest.raises(RuntimeError, match="commit failed"): + publish(lifecycle, snapshot) + else: + assert publish(lifecycle, snapshot).graph + finally: + if fail_commit: + event.remove(seeded, "before_commit", fail) + with seeded() as session: + app = session.get(App, "app-1") + version = session.scalar(select(Workflow).where(Workflow.marked_name == "Release")) + assert app is not None + assert (version is None) is fail_commit + if fail_commit: + assert app.workflow_id == "published" + else: + assert version is not None + assert app.workflow_id == version.id + + +@pytest.mark.parametrize( + ("workflow_id", "error"), [("draft", DraftWorkflowDeletionError), ("published", WorkflowInUseError)] +) +def test_delete_preserves_domain_rejections( + repository: WorkflowDefinitionRepository, workflow_id: str, error: type[Exception] +) -> None: + with pytest.raises(error): + repository.delete(CONTEXT, WorkflowOwner("app-1"), workflow_id) + + +def test_sync_prepares_patch_without_replacing_server_environment(seeded: sessionmaker[Session]) -> None: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.features = '{"collaborator": true}' + draft.environment_variables = [StringVariable(id="untouched", name="OTHER", value="server")] + draft_hash = draft.unique_hash + command = DraftSyncCommand( + graph={"nodes": [], "edges": []}, + features={}, + unique_hash=draft_hash, + is_collaborative=True, + environment_upserts=[{"id": "id", "name": "KEY", "value_type": "string", "value": "value"}], + environment_deletions=[], + conversation_variables=[], + ) + application( + WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + ).sync(CONTEXT, "app-1", command) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert draft.features_dict["collaborator"] is True + assert {value.name: value.value for value in draft.environment_variables} == {"OTHER": "server", "KEY": "value"} + application( + WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + ).sync(CONTEXT, "app-1", replace(command, environment_upserts=None)) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert {value.name: value.value for value in draft.environment_variables} == {"OTHER": "server", "KEY": "value"} + + +def test_app_maintainers_reuses_tenant_scoped_app_repository(seeded: sessionmaker[Session]) -> None: + with seeded.begin() as session: + session.add_all( + [ + make_app(app_id="other", tenant_id="tenant-2"), + make_app(app_id="local", maintainer="owner"), + ] + ) + repository = ConsoleAppRepository(session_factory=seeded) + assert repository.maintainers(CONTEXT, ["app-1", "other", "local", "missing"]) == {"app-1": None, "local": "owner"} + + +@pytest.mark.parametrize("operation", ["sync", "restore", "publish", "features"]) +@pytest.mark.parametrize("fail_commit", [False, True]) +def test_use_case_notifies_only_after_real_transaction_commits( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, operation: str, fail_commit: bool +) -> None: + repository = WorkflowDefinitionRepository(session_factory=seeded) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + runtime = create_autospec(WorkflowRuntime, instance=True) + service = ConsoleWorkflowService( + conversion=create_autospec(WorkflowConversion, instance=True), + agent_services=WorkflowAgentPublishService, + definitions=repository, + drafts=WorkflowDraftService( + agent_services=WorkflowAgentPublishService, + definitions=WorkflowDraftRepository(seeded), + lifecycle=lifecycle, + retirement=create_autospec(DraftAgentRetirement, instance=True), + ), + lifecycle=lifecycle, + runtime=runtime, + apps=create_autospec(WorkflowAppLookup, instance=True), + presence=create_autospec(WorkflowPresence, instance=True), + access=create_autospec(WorkflowAccess, instance=True), + ) + if operation in {"sync", "restore"}: + with seeded.begin() as session: + session.add( + WorkflowAgentNodeBinding( + id="removed-binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="removed-node", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + agent_id="retired-agent", + node_job_config=WorkflowNodeJobConfig(), + ) + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + before = workflow_snapshot(draft) + steps: list[str] = [] + sessions: list[Session] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def committed(_session: Session) -> None: + steps.append("commit") + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + def retire(context: RequestContext, agent_ids: list[str]) -> None: + assert steps == ["commit"] + assert all(not session.in_transaction() for session in sessions) + assert context == CONTEXT + assert agent_ids == ["retired-agent"] + with seeded() as session: + assert session.get(WorkflowAgentNodeBinding, "removed-binding") is None + steps.append("retire") + + cast(Mock, service._drafts._retirement).retire_unowned.side_effect = lambda **kwargs: retire( + CONTEXT, kwargs["agent_ids"] + ) + + def notify(_app: App, **kwargs: object) -> None: + assert "commit" in steps + assert all(not session.in_transaction() for session in sessions) + notified = kwargs["synced_draft_workflow"] + assert isinstance(notified, Workflow) + with seeded() as session: + persisted = session.get(Workflow, notified.id) + assert persisted is not None + assert persisted.graph == notified.graph + steps.append("notify") + + runtime.retire_agents.side_effect = retire + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "send", notify) + event.listen(seeded, "after_begin", opened) + event.listen(seeded, "after_commit", committed) + if fail_commit: + event.listen(seeded, "before_commit", fail) + + def execute() -> None: + match operation: + case "sync": + service.sync( + CONTEXT, + "app-1", + DraftSyncCommand( + graph={"nodes": [], "edges": [], "changed": True}, + features={}, + unique_hash=before.hash, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + conversation_variables=[], + ), + ) + case "restore": + service.restore(CONTEXT, "app-1", "published") + case "publish": + service.publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + case "features": + service.update_features(CONTEXT, "app-1", {"file_upload": {"enabled": False}}) + + try: + if fail_commit: + with pytest.raises(RuntimeError, match="commit failed"): + execute() + assert steps == [] + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + assert workflow_snapshot(draft) == before + if operation in {"sync", "restore"}: + assert session.get(WorkflowAgentNodeBinding, "removed-binding") is not None + else: + execute() + assert steps == (["commit", "retire", "notify"] if operation in {"sync", "restore"} else ["commit"]) + finally: + event.remove(seeded, "after_begin", opened) + event.remove(seeded, "after_commit", committed) + if fail_commit: + event.remove(seeded, "before_commit", fail) + + +@pytest.mark.parametrize("field", ["graph", "features", "environment_variables"]) +def test_publish_rejects_revision_modified_after_validation(seeded: sessionmaker[Session], field: str) -> None: + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + validated = workflow_snapshot(draft) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if field == "environment_variables": + draft.environment_variables = [StringVariable(id="changed", name="KEY", value="changed")] + else: + setattr(draft, field, '{"changed": true}') + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + with pytest.raises(WorkflowHashNotEqualError): + publish(lifecycle, validated) + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == "published" + + +@pytest.mark.parametrize("node_type", ["trigger-webhook", "trigger-plugin"]) +def test_trigger_limit_rejects_publish_before_version_or_pointer_changes( + seeded: sessionmaker[Session], node_type: str +) -> None: + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": f"trigger-{index}", + "data": {"type": node_type, "title": "Trigger", "subscription_id": "sub"}, + } + for index in range(6) + ], + "edges": [], + } + ) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + with pytest.raises(ValueError, match="maximum.*limit"): + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + counter = session.get(WorkflowVersionCounter, "app-1") + assert app is not None + assert app.workflow_id == "published" + assert counter is not None + assert counter.last_version_number == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 2 + + +def test_publication_versions_increase_without_reusing_deleted_version(seeded: sessionmaker[Session]) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + service = application(lifecycle) + for expected in [2, 3, 4]: + service.publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + workflow = session.get(Workflow, app.workflow_id) + assert workflow is not None + assert workflow.version_number == expected + if expected == 3: + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.workflow_id = "published" + service.delete(CONTEXT, WorkflowOwner("app-1"), workflow.id) + + +@pytest.mark.parametrize( + "invalid_configuration", ["missing-draft", "missing-model", "model-mode", "schedule", "cloud-limit"] +) +def test_publication_retains_blocking_domain_validations( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + config_overrides: Callable[..., None], + invalid_configuration: str, +) -> None: + expected_error: type[Exception] = ValueError + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + if invalid_configuration == "missing-draft": + session.delete(draft) + elif invalid_configuration in {"missing-model", "model-mode"}: + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "llm", + "data": { + "type": "llm", + "model_selector": ["env", "shared_model"], + "model": { + "provider": "provider", + "name": "model", + "mode": "chat", + "completion_params": {}, + }, + }, + } + ], + "edges": [], + } + ) + if invalid_configuration == "model-mode": + draft.environment_variables = [ + LLMEnvironmentVariable( + name="shared_model", value={"provider": "provider", "name": "model", "mode": "completion"} + ) + ] + elif invalid_configuration == "schedule": + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "schedule", + "data": { + "title": "Schedule", + "type": "trigger-schedule", + "mode": "cron", + "cron_expression": "invalid", + "timezone": "UTC", + }, + } + ], + "edges": [], + } + ) + else: + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + monkeypatch.setattr( + workflow_module.BillingService, "get_info", lambda *_args: builders.subscription_info("sandbox") + ) + draft.graph = json.dumps( + {"nodes": [{"id": str(index), "data": {"type": "trigger-webhook"}} for index in range(3)], "edges": []} + ) + expected_error = TriggerNodeLimitExceededError + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + with pytest.raises(expected_error): + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Rejected", marked_comment="") + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == "published" + assert session.scalar(select(Workflow.id).where(Workflow.marked_name == "Rejected")) is None + + +def test_restore_preserves_historical_features_without_normalizing_persisted_source( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + features = json.dumps( + { + "file_upload": { + "image": {"enabled": True, "number_limits": 6, "transfer_methods": ["remote_url", "local_file"]} + } + } + ) + with seeded.begin() as session: + workflow = session.get(Workflow, "published") + assert workflow is not None + workflow.features = features + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "send", lambda *_args, **_kwargs: None) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + application(lifecycle).restore(CONTEXT, "app-1", "published") + with seeded() as session: + for workflow_id in ["draft", "published"]: + workflow = session.get(Workflow, workflow_id) + assert workflow is not None + assert workflow.serialized_features == features + + +def test_publication_serializes_on_app_and_closes_the_transaction( + seeded: sessionmaker[Session], +) -> None: + locks: list[Session] = [] + + def capture(execution: ORMExecuteState) -> None: + if not isinstance(execution.statement, Select): + return + sql = str(execution.statement.compile(dialect=postgresql.dialect())) + if "FROM apps" in sql and "FOR UPDATE" in sql: + locks.append(execution.session) + + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + event.listen(seeded, "do_orm_execute", capture) + try: + application(lifecycle).publish(CONTEXT, "app-1", marked_name="Release", marked_comment="") + finally: + event.remove(seeded, "do_orm_execute", capture) + assert len(locks) == 1 + assert all(not session.in_transaction() for session in locks) + + +@pytest.mark.parametrize("over_limit", [False, True]) +def test_migration_uses_the_same_atomic_publication_boundary( + seeded: sessionmaker[Session], over_limit: bool, *, workflow_tools: WorkflowToolManageService +) -> None: + app_id = str(uuid4()) + with seeded.begin() as session: + session.add(make_app(app_id=app_id, mode=AppMode.WORKFLOW, created_by="account-1")) + session.add( + make_workflow( + app_id=app_id, + graph={ + "nodes": [ + {"id": str(index), "data": {"type": "trigger-webhook", "title": "Webhook"}} + for index in range(6 if over_limit else 1) + ], + "edges": [], + }, + ) + ) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + migration = MigrationImportService(workflows=application(lifecycle), app_dsl=Mock(), workflow_tools=workflow_tools) + with seeded() as session: + actor = session.get(Account, "account-1") + assert actor is not None + target = ImportTarget("tenant-1", "Test", actor.id, actor.email) + if over_limit: + with pytest.raises(ValueError, match="maximum webhook node limit"): + migration._ensure_workflow_app_is_published(target, actor, app_id, session=session) + else: + migration._ensure_workflow_app_is_published(target, actor, app_id, session=session) + app = session.get(App, app_id) + assert app is not None + if over_limit: + assert app.workflow_id is None + else: + assert app.workflow_id is not None + assert session.get(Workflow, app.workflow_id) is not None + from models.trigger import WorkflowWebhookTrigger + + assert ( + session.scalar(select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app_id)) + is not None + ) + + +def configure_runtime_draft(factory: sessionmaker[Session], name: str) -> None: + with factory.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + {"id": f"webhook-{name}", "data": {"type": "trigger-webhook", "title": name}}, + { + "id": f"plugin-{name}", + "data": { + "type": "trigger-plugin", + "title": name, + "provider_id": "test/provider", + "event_name": "created", + "subscription_id": f"subscription-{name}", + }, + }, + { + "id": f"schedule-{name}", + "data": { + "type": "trigger-schedule", + "title": name, + "mode": "cron", + "cron_expression": "0 * * * *", + "timezone": "UTC", + }, + }, + ], + "edges": [], + } + ) + + +def assert_runtime_configuration(factory: sessionmaker[Session], name: str) -> str: + from models.trigger import AppTrigger, WorkflowPluginTrigger, WorkflowSchedulePlan, WorkflowWebhookTrigger + + with factory() as session: + app = session.get(App, "app-1") + assert app is not None + version = session.get(Workflow, app.workflow_id) + assert version is not None + assert version.marked_name == name + assert set(session.scalars(select(AppTrigger.node_id).where(AppTrigger.app_id == app.id))) == { + f"webhook-{name}", + f"plugin-{name}", + f"schedule-{name}", + } + webhook = session.scalars(select(WorkflowWebhookTrigger).where(WorkflowWebhookTrigger.app_id == app.id)).one() + assert webhook.node_id == f"webhook-{name}" + assert len(webhook.webhook_id) == 24 + plugin = session.scalars(select(WorkflowPluginTrigger).where(WorkflowPluginTrigger.app_id == app.id)).one() + assert plugin.node_id == f"plugin-{name}" + assert plugin.subscription_id == f"subscription-{name}" + schedule = session.scalars(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == app.id)).one() + assert schedule.node_id == f"schedule-{name}" + assert schedule.next_run_at is not None + assert schedule.cron_expression == "0 * * * *" + return version.id + + +def test_redis_outage_cannot_leave_a_successful_publication_without_runtime_configuration( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from redis.exceptions import ConnectionError as RedisConnectionError + + from extensions.ext_redis import redis_client + + configure_runtime_draft(seeded, "A") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + + def unavailable() -> None: + raise RedisConnectionError("Redis temporarily unavailable") + + monkeypatch.setattr(redis_client, "_require_client", unavailable) + application(lifecycle).publish(CONTEXT, "app-1", marked_name="A", marked_comment="") + assert_runtime_configuration(seeded, "A") + + +def test_trigger_storage_failure_rolls_back_version_pointer_and_every_projection( + seeded: sessionmaker[Session], +) -> None: + from models.trigger import WorkflowSchedulePlan + + configure_runtime_draft(seeded, "A") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + service = application(lifecycle) + service.publish(CONTEXT, "app-1", marked_name="A", marked_comment="") + original_version = assert_runtime_configuration(seeded, "A") + configure_runtime_draft(seeded, "B") + + def reject_schedule_update(session: Session, _context: object, _instances: object) -> None: + # This fails after webhook/plugin writes have flushed, exercising real rollback. + for value in session.dirty: + if isinstance(value, WorkflowSchedulePlan) and value.node_id == "schedule-B": + raise RuntimeError("schedule storage unavailable") + + event.listen(seeded, "before_flush", reject_schedule_update) + try: + with pytest.raises(RuntimeError, match="schedule storage unavailable"): + service.publish(CONTEXT, "app-1", marked_name="B", marked_comment="") + finally: + event.remove(seeded, "before_flush", reject_schedule_update) + assert assert_runtime_configuration(seeded, "A") == original_version + with seeded() as session: + assert session.scalar(select(Workflow.id).where(Workflow.marked_name == "B")) is None + counter = session.get(WorkflowVersionCounter, "app-1") + assert counter is not None + assert counter.last_version_number == 2 + + +def test_republication_replaces_all_trigger_configuration( + seeded: sessionmaker[Session], +) -> None: + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + for name in ["A", "B"]: + configure_runtime_draft(seeded, name) + application(lifecycle).publish(CONTEXT, "app-1", marked_name=name, marked_comment="") + assert_runtime_configuration(seeded, name) + + +def test_draft_webhook_receiver_uses_injected_database( + seeded: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from events.event_handlers.sync_webhook_when_app_created import handle + from models.trigger import WorkflowWebhookTrigger + from services.trigger import webhook_service + + configure_runtime_draft(seeded, "draft") + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + monkeypatch.setattr(webhook_service, "db", object()) + monkeypatch.setattr(gateway_module.app_draft_workflow_was_synced, "receivers_for", lambda _app: iter([handle])) + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + lifecycle.draft_synced(CONTEXT, WorkflowOwner("app-1"), snapshot) + with seeded() as session: + assert session.scalars(select(WorkflowWebhookTrigger.node_id)).all() == ["webhook-draft"] + + +@pytest.mark.parametrize("retained", [set(), {"kept"}]) +def test_dataset_sync_removes_every_duplicate_stale_relationship( + seeded: sessionmaker[Session], retained: set[str] +) -> None: + with seeded.begin() as session: + session.add_all( + [AppDatasetJoin(app_id="app-1", dataset_id=value) for value in ["removed", "removed", "kept", "kept"]] + ) + session.add(AppDatasetJoin(app_id="other-app", dataset_id="removed")) + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + WorkflowDefinitionStore.sync_dataset_relationships(session, app, retained) + with seeded() as session: + assert ( + set(session.scalars(select(AppDatasetJoin.dataset_id).where(AppDatasetJoin.app_id == "app-1"))) == retained + ) + assert ( + session.scalars(select(AppDatasetJoin).where(AppDatasetJoin.app_id == "other-app")).one().dataset_id + == "removed" + ) + + +@pytest.mark.parametrize("field", ["environment", "conversation"]) +def test_variable_updates_lock_the_owned_draft_and_close_their_transaction( + seeded: sessionmaker[Session], field: str +) -> None: + repository = WorkflowDraftRepository(seeded) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None + sessions: list[Session] = [] + locked_statements: list[str] = [] + + def capture(execution: ORMExecuteState) -> None: + if not isinstance(execution.statement, Select): + return + statement = str(execution.statement.compile(dialect=postgresql.dialect())) + if "FOR UPDATE" in statement: + sessions.append(execution.session) + locked_statements.append(statement) + + event.listen(seeded, "do_orm_execute", capture) + variables = [StringVariable(id="changed", name="changed", value="new")] + try: + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=snapshot, + environment_variables=prepare_environment_variables( + tenant_id="tenant-1", source=snapshot, variables=variables + ).value + if field != "conversation" + else None, + conversation_variables=variables if field == "conversation" else None, + ) + finally: + event.remove(seeded, "do_orm_execute", capture) + assert len(locked_statements) == 2 + assert "FROM apps" in locked_statements[0] + assert all( + column in locked_statements[1] for column in ["workflows.app_id", "workflows.tenant_id", "workflows.version"] + ) + assert all(not session.in_transaction() for session in sessions) + with seeded() as session: + workflow = session.get(Workflow, "draft") + assert workflow is not None + actual = workflow.conversation_variables if field == "conversation" else workflow.environment_variables + assert [(v.id, v.value) for v in actual] == [("changed", "new")] + assert workflow.updated_by == "account-1" + assert workflow.updated_at is not None + + +def test_variable_update_does_not_commit_an_ambient_session(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + with seeded() as caller: + caller.add(make_account(account_id="uncommitted", email="uncommitted@example.com")) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=snapshot, + environment_variables="{}", + conversation_variables=None, + ) + with seeded() as reader: + assert reader.get(Account, "uncommitted") is None + assert caller.new + caller.rollback() + + +def test_variable_update_commit_failure_rolls_back_all_fields(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + with seeded() as session: + before = session.get(Workflow, "draft") + assert before is not None + snapshot = workflow_snapshot(before) + + def fail_commit(_session: Session) -> None: + raise RuntimeError("commit failed") + + event.listen(seeded, "before_commit", fail_commit) + try: + with pytest.raises(RuntimeError, match="commit failed"): + repository.update_draft_variables( + CONTEXT, + "app-1", + expected=snapshot, + environment_variables=prepare_environment_variables( + tenant_id="tenant-1", + source=snapshot, + variables=[StringVariable(id="new-env", name="key", value="value")], + ).value, + conversation_variables=[StringVariable(id="new-conv", name="topic", value="value")], + ) + finally: + event.remove(seeded, "before_commit", fail_commit) + with seeded() as session: + after = session.get(Workflow, "draft") + assert after is not None + assert workflow_snapshot(after) == snapshot + + +def test_variable_update_rejects_another_tenant(seeded: sessionmaker[Session]) -> None: + repository = WorkflowDraftRepository(seeded) + snapshot = repository.snapshot(CONTEXT, WorkflowOwner("app-1")) + assert snapshot is not None + with pytest.raises(ConsoleAppNotFoundError): + repository.update_draft_variables( + CONTEXT._replace(active_workspace_id="other-tenant"), + "app-1", + expected=snapshot, + environment_variables="{}", + conversation_variables=None, + ) + + +def test_republish_without_schedule_removes_the_runtime_plan(seeded: sessionmaker[Session]) -> None: + from models.trigger import WorkflowSchedulePlan + + configure_runtime_draft(seeded, "scheduled") + lifecycle = WorkflowDefinitionGateway( + session_factory=seeded, + drafts=WorkflowDraftRepository(seeded), + definitions=WorkflowDefinitionRepository(session_factory=seeded), + workflows=WorkflowService(session_maker=seeded), + credentials=CredentialQueryRepository(session_factory=seeded), + skills=create_autospec(AgentSkillReader, instance=True), + ) + service = application(lifecycle) + service.publish(CONTEXT, "app-1", marked_name="Scheduled", marked_comment="") + with seeded.begin() as session: + assert session.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == "app-1")) is not None + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps({"nodes": [], "edges": []}) + service.publish(CONTEXT, "app-1", marked_name="Unscheduled", marked_comment="") + with seeded() as session: + assert session.scalar(select(WorkflowSchedulePlan).where(WorkflowSchedulePlan.app_id == "app-1")) is None + + +@pytest.mark.parametrize("rollback_at", [None, "knowledge", "after-activation"]) +def test_agent_bindings_knowledge_spaces_and_active_version_share_the_publication_transaction( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rollback_at: str | None +) -> None: + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig + from services.workflow.publication_service import prepare_publication, publish_workflow + + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "agent-node", + "data": { + "type": "agent", + "version": "2", + "agent_node_kind": "dify_agent", + }, + } + ], + "edges": [], + } + ) + session.add_all( + [ + Agent( + id="roster", + tenant_id="tenant-1", + name="Roster", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="agent-config", + ), + AgentConfigSnapshot( + id="agent-config", + tenant_id="tenant-1", + agent_id="roster", + version=1, + config_snapshot=AgentSoulConfig( + model=AgentSoulModelConfig( + plugin_id="langgenius/openai", + model_provider="openai", + model="test-model", + ) + ), + ), + WorkflowAgentNodeBinding( + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent-node", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="roster", + current_snapshot_id="agent-config", + node_job_config={}, + created_by="account-1", + ), + ] + ) + with seeded() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + snapshot = workflow_snapshot(draft) + repository = WorkflowDefinitionRepository(session_factory=seeded) + published_id = "" + knowledge_sync_sessions: list[Session] = [] + + def sync_knowledge_spaces(*, session: Session, session_maker: sessionmaker[Session], workflow: Workflow) -> None: + nonlocal published_id + assert session_maker is seeded + published_id = workflow.id + published_binding = session.scalar( + select(WorkflowAgentNodeBinding).where( + WorkflowAgentNodeBinding.tenant_id == workflow.tenant_id, + WorkflowAgentNodeBinding.app_id == workflow.app_id, + WorkflowAgentNodeBinding.workflow_id == workflow.id, + WorkflowAgentNodeBinding.workflow_version == workflow.version, + ) + ) + assert published_binding is not None + app = session.get(App, workflow.app_id) + assert app is not None + app.description = "KnowledgeFS bindings synchronized" + knowledge_sync_sessions.append(session) + if rollback_at == "knowledge": + raise RuntimeError("rollback during KnowledgeFS synchronization") + + monkeypatch.setattr( + "services.agent.knowledge_spaces.sync_workflow_app_knowledge_bindings", + sync_knowledge_spaces, + ) + + with repository.binding_reader() as bindings: + agents = WorkflowAgentPublishService(repository=bindings).publication_state(draft_workflow=snapshot) + + def publish() -> None: + nonlocal published_id + with repository.publication(CONTEXT, "app-1", snapshot) as transaction: + published = publish_workflow( + transaction, + prepare_publication(snapshot), + agent_service=WorkflowAgentPublishService(repository=transaction.bindings), + agents=agents, + marked_name="Agents", + marked_comment="", + ) + published_id = published.id + if rollback_at == "after-activation": + raise RuntimeError("rollback after binding copy and activation") + + should_rollback = rollback_at is not None + if should_rollback: + with pytest.raises(RuntimeError, match="rollback"): + publish() + else: + publish() + assert len(knowledge_sync_sessions) == 1 + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == ("published" if should_rollback else published_id) + assert app.description == ("" if should_rollback else "KnowledgeFS bindings synchronized") + assert (session.get(Workflow, published_id) is None) is should_rollback + bindings = list( + session.scalars( + select(WorkflowAgentNodeBinding).where( + WorkflowAgentNodeBinding.workflow_id == published_id, + ) + ) + ) + assert len(bindings) == (0 if should_rollback else 1) + if bindings: + assert bindings[0].current_snapshot_id == "agent-config" + assert ( + session.scalar( + select(func.count()) + .select_from(WorkflowAgentNodeBinding) + .where( + WorkflowAgentNodeBinding.workflow_id == "draft", + ) + ) + == 1 + ) + + +@pytest.mark.parametrize("concurrent_edit", [None, "graph", "binding", "agent-snapshot"]) +def test_skill_archive_validation_releases_connections_and_rechecks_agent_state( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: str | None +) -> None: + import io + import zipfile + + from sqlalchemy.engine import Engine + from sqlalchemy.pool import QueuePool + + from extensions.application_services.workflow import build_workflow_definition_gateway + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, AgentSoulPromptConfig + from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest + from models.tools import ToolFile + + soul = AgentSoulConfig( + model=AgentSoulModelConfig(plugin_id="langgenius/openai", model_provider="openai", model="test"), + prompt=AgentSoulPromptConfig(system_prompt="Use [§skill:archived-skill§]"), + ) + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}], + "edges": [], + } + ) + archive = ToolFile( + tenant_id="tenant-1", + user_id="account-1", + conversation_id=None, + file_key="skills/published.zip", + mimetype="application/zip", + ) + session.add_all( + [ + archive, + Agent( + id="agent", + tenant_id="tenant-1", + name="Agent", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="soul", + ), + AgentConfigSnapshot(id="soul", tenant_id="tenant-1", agent_id="agent", version=1, config_snapshot=soul), + AgentConfigSnapshot( + id="new-soul", tenant_id="tenant-1", agent_id="agent", version=2, config_snapshot=soul + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="agent", + current_snapshot_id="soul", + node_job_config={}, + created_by="account-1", + ), + Skill( + id="skill", + tenant_id="tenant-1", + name="draft-name", + display_name="Skill", + latest_published_version_id="version", + ), + SkillVersion( + id="version", + skill_id="skill", + version_number=1, + manifest=SkillVersionManifest(files=[]), + archive_tool_file_id=archive.id, + hash_code="hash", + archive_size=123, + ), + AgentSkillBindingSnapshot( + tenant_id="tenant-1", agent_id="agent", config_snapshot_id="soul", skill_id="skill", priority=0 + ), + ] + ) + archive_bytes = io.BytesIO() + with zipfile.ZipFile(archive_bytes, "w") as zip_file: + zip_file.writestr("SKILL.md", "---\nname: archived-skill\ndescription: Published skill\n---\nInstructions") + sessions: list[Session] = [] + statements: list[str] = [] + downloads: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def query(execution: ORMExecuteState) -> None: + if isinstance(execution.statement, Select): + statements.append(str(execution.statement.compile(dialect=postgresql.dialect()))) + + def load(key: str) -> bytes: + assert all(not session.in_transaction() for session in sessions) + engine = seeded.kw["bind"] + assert isinstance(engine, Engine) + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + assert not any("FOR UPDATE" in statement for statement in statements) + downloads.append(key) + if concurrent_edit: + with seeded.begin() as session: + if concurrent_edit == "graph": + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + elif concurrent_edit == "binding": + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + else: + agent = session.get(Agent, "agent") + assert agent is not None + agent.active_config_snapshot_id = "new-soul" + return archive_bytes.getvalue() + + monkeypatch.setattr("services.skill_management_service.storage.load_once", load) + build_console_workflow_variables( + database_client=seeded, variables=build_workflow_variable_service(database_client=seeded) + ) + with seeded.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = AppMode.ADVANCED_CHAT + workflows = WorkflowService(seeded) + monkeypatch.setattr(workflows, "validate_graph_structure", lambda **_kwargs: None) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + service = application( + build_workflow_definition_gateway(seeded, definitions, workflows, drafts=WorkflowDraftRepository(seeded)) + ) + event.listen(seeded, "after_begin", opened) + event.listen(seeded, "do_orm_execute", query) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish(CONTEXT, "app-1", marked_name="Skill", marked_comment="") + else: + service.publish(CONTEXT, "app-1", marked_name="Skill", marked_comment="") + finally: + event.remove(seeded, "after_begin", opened) + event.remove(seeded, "do_orm_execute", query) + assert downloads == ["skills/published.zip"] + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert (app.workflow_id == "published") is bool(concurrent_edit) + versions = session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) + assert versions == (1 if concurrent_edit else 2) + + +@pytest.mark.parametrize("rollback", [False, True]) +@pytest.mark.parametrize("packaged", [False, True]) +def test_dsl_draft_and_app_share_outer_commit_and_effects( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, rollback: bool, packaged: bool +) -> None: + from extensions.application_services.workflow import build_app_dsl_service + from services.agent.dsl_service import AgentDslService + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.entities.dsl_entities import DslImportWarning + + effects: list[object] = [] + locks: list[str] = [] + + def query(execution: ORMExecuteState) -> None: + statement = execution.statement + if ( + isinstance(statement, Select) + and Workflow.__table__ in statement.get_final_froms() + and "FOR UPDATE" in str(statement.compile(dialect=postgresql.dialect())) + ): + locks.append(str(statement.compile(dialect=postgresql.dialect()))) + + def retire(**kwargs: object) -> None: + assert not outer.in_transaction() + effects.append(kwargs["agent_ids"]) + + def notify(_app: App, **kwargs: object) -> None: + assert not outer.in_transaction() + workflow = kwargs["synced_draft_workflow"] + assert isinstance(workflow, Workflow) + effects.append(workflow.graph_dict) + + def materialize( + _service: AgentDslService, **kwargs: object + ) -> tuple[dict[str, object], list[DslImportWarning], set[str]]: + assert outer.in_transaction() + workflow = kwargs["workflow"] + assert isinstance(workflow, WorkflowSnapshot) + assert workflow.app_id == "app-1" + # A dependent import write must be part of the same transaction. + outer.add(make_workflow(workflow_id="import-dependent", app_id="other-app")) + return {"nodes": [], "edges": [], "materialized": True}, [], {"removed-agent"} + + monkeypatch.setattr(WorkflowAgentRetirementService, "retire_unowned", staticmethod(retire)) + monkeypatch.setattr(AgentDslService, "import_workflow_packages", materialize) + event.listen(seeded, "do_orm_execute", query) + gateway_module.app_draft_workflow_was_synced.connect(notify, weak=False) + try: + with seeded() as outer: + app, account = outer.get(App, "app-1"), outer.get(Account, "account-1") + assert app is not None + assert account is not None + build_app_dsl_service(outer)._create_or_update_app( + app=app, + account=account, + data={ + "app": {"mode": "workflow", "name": "Imported"}, + "workflow": {"graph": {"nodes": [], "edges": [], "imported": True}}, + "agent_packages": {"package": {}} if packaged else {}, + }, + ) + assert effects == [] + assert len(locks) == 1 + if rollback: + outer.rollback() + # Reusing the session must not resurrect rolled-back effects. + outer.commit() + else: + outer.commit() + assert effects == ( + [] + if rollback + else [ + ["removed-agent"] if packaged else [], + {"nodes": [], "edges": [], "materialized" if packaged else "imported": True}, + ] + ) + finally: + event.remove(seeded, "do_orm_execute", query) + gateway_module.app_draft_workflow_was_synced.disconnect(notify) + with seeded() as session: + app, draft = session.get(App, "app-1"), session.get(Workflow, "draft") + assert app is not None + assert draft is not None + assert (app.name == "Imported") is not rollback + assert bool(draft.graph_dict.get("materialized" if packaged else "imported")) is not rollback + assert (session.get(Workflow, "import-dependent") is not None) is (packaged and not rollback) + + +@pytest.mark.parametrize("state", ["new", "replace", "stale", "collaborative"]) +def test_shared_draft_writer_preserves_hash_and_variable_rules(seeded: sessionmaker[Session], state: str) -> None: + from extensions.application_services.workflow import build_workflow_definition_gateway + + with seeded.begin() as session: + draft = session.get(Workflow, "draft") + assert draft is not None + draft.environment_variables = [StringVariable(id="server", name="SERVER", value="server")] + draft.features = '{"server": true}' + previous_hash = draft.unique_hash + if state == "new": + session.delete(draft) + definitions = WorkflowDefinitionRepository(session_factory=seeded) + lifecycle = build_workflow_definition_gateway( + seeded, definitions, WorkflowService(seeded), drafts=WorkflowDraftRepository(seeded) + ) + service = application(lifecycle) + command = DraftSyncCommand( + graph={"nodes": [], "edges": [], "edited": True}, + features={"client": True}, + unique_hash="stale" if state == "stale" else previous_hash, + is_collaborative=state == "collaborative", + environment_upserts=None, + environment_deletions=[], + conversation_variables=[], + environment_variables=[{"id": "client", "name": "CLIENT", "value_type": "string", "value": "client"}], + ) + if state == "stale": + with pytest.raises(WorkflowHashNotEqualError): + service.sync(CONTEXT, "app-1", command) + else: + result = service.sync(CONTEXT, "app-1", command) + assert result.hash != previous_hash + with seeded() as session: + draft = session.scalar(select(Workflow).where(Workflow.app_id == "app-1", Workflow.version == "draft")) + assert draft is not None + preserve = state in {"stale", "collaborative"} + assert draft.features_dict.get("server", False) is preserve + assert draft.features_dict.get("client", False) is not preserve + assert {variable.name: variable.value for variable in draft.environment_variables} == ( + {"SERVER": "server"} if preserve else {"CLIENT": "client"} + ) + assert bool(draft.graph_dict.get("edited")) is (state != "stale") + + +@pytest.mark.parametrize("snippet", [False, True]) +def test_publication_repository_is_independent_of_agent_service_and_commits_requested_operations( + seeded: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, snippet: bool +) -> None: + from models.snippet import CustomizedSnippet, SnippetType + from models.workflow import WorkflowKind + from repositories.agent.workflow_binding_repository import WorkflowAgentBindingRepository + from repositories.workflow.snippet_publication_repository import SnippetPublicationRepository + + # A repository must not instantiate or invoke Agent business orchestration, + # even if it used to receive that service hidden behind a factory/Protocol. + monkeypatch.setattr( + WorkflowAgentPublishService, "__init__", Mock(side_effect=AssertionError("repository called service")) + ) + if snippet: + with seeded.begin() as session: + session.add( + CustomizedSnippet( + id="app-1", tenant_id="tenant-1", name="Snippet", type=SnippetType.NODE, created_by="account-1" + ) + ) + draft = session.get(Workflow, "draft") + assert draft is not None + draft.kind = WorkflowKind.SNIPPET + snippets = SnippetPublicationRepository(seeded) + with snippets.draft(tenant_id="tenant-1", snippet_id="app-1") as (source, bindings): + assert isinstance(bindings, WorkflowAgentBindingRepository) + with snippets.publication(source) as transaction: + workflow = transaction.create_version(account_id="account-1") + transaction.activate(workflow, account_id="account-1") + with seeded() as session: + row = session.get(CustomizedSnippet, "app-1") + assert row is not None + assert row.is_published is True + assert session.get(Workflow, workflow.id) is not None + else: + definitions = WorkflowDefinitionRepository(session_factory=seeded) + with definitions.draft(CONTEXT, "app-1") as read: + assert read is not None + assert isinstance(read.bindings, WorkflowAgentBindingRepository) + source = read.workflow + with definitions.publication(CONTEXT, "app-1", source) as publication: + created = publication.create_version(marked_name="Published", marked_comment="") + publication.activate(created) + assert isinstance(publication.bindings, WorkflowAgentBindingRepository) + with seeded() as session: + app = session.get(App, "app-1") + assert app is not None + assert app.workflow_id == created.id + assert session.get(Workflow, created.id) is not None diff --git a/api/tests/unit_tests/repositories/workflow/test_draft_repository.py b/api/tests/unit_tests/repositories/workflow/test_draft_repository.py new file mode 100644 index 00000000000000..8dc0484c5c3b7d --- /dev/null +++ b/api/tests/unit_tests/repositories/workflow/test_draft_repository.py @@ -0,0 +1,446 @@ +"""The shared draft use case commits each owner's state in one short transaction.""" + +import json +from collections.abc import Iterable +from dataclasses import replace +from typing import Literal +from uuid import uuid4 + +import pytest +from sqlalchemy import Select, event, select +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import ORMExecuteState, Session, sessionmaker + +from extensions.application_services.workflow import build_workflow_drafts +from extensions.ext_application_services import ApplicationServices +from graphon.variables import StringVariable +from machinery.context import RequestContext +from models.account import TenantAccountJoin, TenantAccountRole +from models.dataset import Pipeline +from models.snippet import CustomizedSnippet +from models.workflow import Workflow, WorkflowKind, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError +from services.workflow.contracts import DraftSyncCommand, WorkflowOwner +from tests.helpers.unit.model_factories import make_account, make_tenant, make_workflow + +CONTEXT = RequestContext("draft", None, "account-1", "tenant-1") + + +@pytest.fixture(params=["snippet", "pipeline"]) +def owner(request: pytest.FixtureRequest, sqlite_session_factory: sessionmaker[Session]) -> WorkflowOwner: + kind: Literal["snippet", "pipeline"] = request.param + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_account(account_id=CONTEXT.account_id), + make_tenant(tenant_id=CONTEXT.active_workspace_id), + TenantAccountJoin( + tenant_id=CONTEXT.active_workspace_id, account_id=CONTEXT.account_id, role=TenantAccountRole.EDITOR + ), + ] + ) + if kind == "snippet": + session.add( + CustomizedSnippet( + id="owner", + tenant_id=CONTEXT.active_workspace_id, + name="Snippet", + type="node", + created_by=CONTEXT.account_id, + ) + ) + else: + pipeline = Pipeline(tenant_id=CONTEXT.active_workspace_id, name="Pipeline", created_by=CONTEXT.account_id) + pipeline.id = "owner" + session.add(pipeline) + return WorkflowOwner("owner", kind) + + +def command() -> DraftSyncCommand: + return DraftSyncCommand( + graph={"nodes": [{"id": "llm", "data": {"type": "llm"}}], "edges": []}, + features={}, + unique_hash=None, + is_collaborative=False, + environment_upserts=None, + environment_deletions=[], + environment_variables=[], + conversation_variables=[], + input_fields=[{"variable": "query"}], + rag_pipeline_variables=[ + {"variable": "source", "belong_to_node_id": "llm", "type": "text-input", "label": "Source"} + ], + ) + + +def test_create_update_and_conflict_are_shared( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session] +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + draft = service.sync(CONTEXT, owner, command()) + assert draft.kind == ("snippet" if owner.kind == "snippet" else "standard") + assert draft.type == ("workflow" if owner.kind == "snippet" else "rag-pipeline") + with sqlite_session_factory() as session: + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == [{"variable": "query"}] + else: + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == draft.id + assert next(iter(json.loads(draft.rag_pipeline_variables).values()))["variable"] == "source" + updated = service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.hash, graph={"nodes": [], "edges": []})) + assert updated.id == draft.id + assert updated.hash != draft.hash + assert updated.updated_by == CONTEXT.account_id + with pytest.raises(WorkflowHashNotEqualError): + service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.hash, input_fields=[])) + with sqlite_session_factory() as session: + row = session.get(Workflow, draft.id) + assert row is not None + assert row.graph_dict == updated.graph_dict + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == [{"variable": "query"}] + + +@pytest.mark.parametrize("existing", [False, True]) +def test_owner_and_draft_roll_back_together( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], existing: bool +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + before = service.sync(CONTEXT, owner, command()) if existing else None + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + event.listen(sqlite_session_factory, "before_commit", fail) + try: + with pytest.raises(RuntimeError, match="commit failed"): + service.sync( + CONTEXT, + owner, + replace(command(), unique_hash=before.hash if before else None, graph={"nodes": []}, input_fields=[]), + ) + finally: + event.remove(sqlite_session_factory, "before_commit", fail) + with sqlite_session_factory() as session: + draft = session.scalar(select(Workflow).where(Workflow.app_id == owner.id)) + assert (workflow_snapshot(draft) if draft else None) == before + if owner.kind == "snippet": + snippet = session.get(CustomizedSnippet, owner.id) + assert snippet is not None + assert snippet.input_fields_list == ([{"variable": "query"}] if existing else []) + else: + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == (before.id if before else None) + + +@pytest.mark.parametrize("existing", [False, True]) +def test_restore_copies_stored_values_and_sets_new_draft_pointer( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], existing: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + service = build_workflow_drafts(sqlite_session_factory) + before = service.sync(CONTEXT, owner, command()) if existing else None + with sqlite_session_factory.begin() as session: + published = make_workflow( + workflow_id="published", + app_id=owner.id, + version="1", + graph={"nodes": [{"id": "restored", "data": {"type": "llm"}}], "edges": []}, + ) + published.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + published.type = WorkflowType.WORKFLOW if owner.kind == "snippet" else WorkflowType.RAG_PIPELINE + published.features = '{"some_preserved_feature": true}' + if owner.kind == "pipeline": + variable_id = str(uuid4()) + published._environment_variables = json.dumps( + {variable_id: {"id": variable_id, "name": "secret", "value_type": "secret", "value": "ciphertext"}} + ) + session.add(published) + + def no_key_io(*_args: object, **_kwargs: object) -> str: + pytest.fail("Restoring stored variables must not contact the key service") + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", no_key_io) + monkeypatch.setattr("core.helper.encrypter.encrypt_token", no_key_io) + locked_sources: list[str] = [] + + def capture(query: ORMExecuteState) -> None: + if isinstance(query.statement, Select): + compiled = query.statement.compile(dialect=postgresql.dialect()) + if "FOR UPDATE" in str(compiled) and "published" in compiled.params.values(): + locked_sources.append(str(compiled)) + + event.listen(sqlite_session_factory, "do_orm_execute", capture) + try: + restored = service.restore(CONTEXT, owner, "published") + finally: + event.remove(sqlite_session_factory, "do_orm_execute", capture) + assert locked_sources, "The current restore use case must lock its published source" + + assert restored.version == "draft" + assert restored.graph == published.graph + assert restored.features == published.features + assert restored.environment_variables == published._environment_variables + assert restored.kind == published.kind + if before: + assert restored.id == before.id + with sqlite_session_factory() as session: + if owner.kind == "pipeline": + pipeline = session.get(Pipeline, owner.id) + assert pipeline is not None + assert pipeline.workflow_id == restored.id + source = session.get(Workflow, "published") + assert source is not None + assert source.graph == published.graph + + +def test_owner_and_version_are_scoped(owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session]) -> None: + service = build_workflow_drafts(sqlite_session_factory) + with pytest.raises(WorkflowNotFoundError): + service.sync(CONTEXT._replace(active_workspace_id="other"), owner, command()) + with pytest.raises(WorkflowNotFoundError): + service.restore(CONTEXT, owner, "missing") + draft = service.sync(CONTEXT, owner, command()) + with pytest.raises(IsDraftWorkflowError): + service.restore(CONTEXT, owner, draft.id) + with sqlite_session_factory.begin() as session: + session.add(make_workflow(workflow_id="foreign", app_id="other", version="1")) + with pytest.raises(WorkflowNotFoundError): + service.restore(CONTEXT, owner, "foreign") + + +def test_snippet_clears_variables_and_rejects_forbidden_graph(sqlite_session_factory: sessionmaker[Session]) -> None: + with sqlite_session_factory.begin() as session: + session.add( + CustomizedSnippet( + id="snippet", + tenant_id=CONTEXT.active_workspace_id, + name="Snippet", + type="node", + created_by=CONTEXT.account_id, + ) + ) + draft = make_workflow(workflow_id="draft", app_id="snippet") + draft.kind = WorkflowKind.SNIPPET + draft.environment_variables = [StringVariable(name="old", value="env")] + draft.conversation_variables = [StringVariable(name="old", value="conversation")] + session.add(draft) + service = build_workflow_drafts(sqlite_session_factory) + owner = WorkflowOwner("snippet", "snippet") + result = service.sync(CONTEXT, owner, replace(command(), unique_hash=draft.unique_hash)) + assert result.environment_variables == result.conversation_variables == "{}" + with pytest.raises(ValueError, match="start-1:start"): + service.sync( + CONTEXT, + owner, + replace( + command(), unique_hash=result.hash, graph={"nodes": [{"id": "start-1", "data": {"type": "start"}}]} + ), + ) + + +@pytest.mark.parametrize("commit_fails", [False, True]) +def test_version_delete_preserves_active_pointer_and_rolls_back( + owner: WorkflowOwner, sqlite_session_factory: sessionmaker[Session], commit_fails: bool +) -> None: + from repositories.workflow.definition_repository import WorkflowDefinitionRepository + from services.errors.workflow_service import WorkflowInUseError + + with sqlite_session_factory.begin() as session: + model = CustomizedSnippet if owner.kind == "snippet" else Pipeline + row = session.get(model, owner.id) + assert row is not None + row.workflow_id = "active" + for workflow_id in ["active", "old"]: + workflow = make_workflow(workflow_id=workflow_id, app_id=owner.id, version=workflow_id) + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + session.add(workflow) + repository = WorkflowDefinitionRepository(session_factory=sqlite_session_factory) + with pytest.raises(WorkflowInUseError): + repository.delete(CONTEXT, owner, "active") + + def fail(_session: Session) -> None: + raise RuntimeError("commit failed") + + if commit_fails: + event.listen(sqlite_session_factory, "before_commit", fail) + try: + if commit_fails: + with pytest.raises(RuntimeError, match="commit failed"): + repository.delete(CONTEXT, owner, "old") + else: + repository.delete(CONTEXT, owner, "old") + finally: + if commit_fails: + event.remove(sqlite_session_factory, "before_commit", fail) + with sqlite_session_factory() as session: + assert (session.get(Workflow, "old") is not None) == commit_fails + assert session.get(Workflow, "active") is not None + + +def test_pipeline_encryption_runs_after_read_session_closes( + sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch +) -> None: + pipeline = Pipeline(tenant_id=CONTEXT.active_workspace_id, name="Pipeline", created_by=CONTEXT.account_id) + with sqlite_session_factory.begin() as session: + session.add(pipeline) + sessions: list[Session] = [] + encrypted: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def encrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == CONTEXT.active_workspace_id + assert sessions + assert all(not session.in_transaction() for session in sessions) + encrypted.append(token) + return "ciphertext" + + monkeypatch.setattr("core.helper.encrypter.encrypt_token", encrypt) + event.listen(sqlite_session_factory, "after_begin", opened) + try: + result = build_workflow_drafts(sqlite_session_factory).sync( + CONTEXT, + WorkflowOwner(pipeline.id, "pipeline"), + replace( + command(), + environment_variables=[ + {"id": str(uuid4()), "name": "secret", "value_type": "secret", "value": "plaintext"} + ], + ), + ) + finally: + event.remove(sqlite_session_factory, "after_begin", opened) + assert encrypted == ["plaintext"] + assert next(iter(json.loads(result.environment_variables).values()))["value"] == "ciphertext" + + +@pytest.mark.parametrize("rollback", [False, True]) +def test_owner_deletion_cleans_bindings_and_retires_after_commit( + owner: WorkflowOwner, + sqlite_session_factory: sessionmaker[Session], + workflow_application: ApplicationServices, + monkeypatch: pytest.MonkeyPatch, + rollback: bool, +) -> None: + from enums.agent import WorkflowAgentBindingType + from models.agent import Agent, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding + + with sqlite_session_factory.begin() as session: + workflow = make_workflow(workflow_id="published", app_id=owner.id, version="v1") + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" else WorkflowKind.STANDARD + session.add_all( + [ + workflow, + Agent( + id="inline", + tenant_id="tenant-1", + name="Inline", + scope=AgentScope.WORKFLOW_ONLY, + source=AgentSource.WORKFLOW, + status=AgentStatus.ACTIVE, + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id=owner.id, + workflow_id=workflow.id, + workflow_version=workflow.version, + node_id="agent", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + agent_id="inline", + current_snapshot_id="snapshot", + node_job_config={}, + ), + ] + ) + dispatched: list[set[str]] = [] + + def collect(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + with sqlite_session_factory() as session: + assert session.get(Workflow, "published") is None + assert session.get(WorkflowAgentNodeBinding, "binding") is None + agent = session.get(Agent, "inline") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + dispatched.append(set(purge_agent_ids)) + + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", collect) + + def reject(_session: Session) -> None: + raise RuntimeError("commit failed") + + if rollback: + event.listen(sqlite_session_factory, "before_commit", reject) + try: + with pytest.raises(RuntimeError, match="commit failed"): + workflow_application.console_workflows.delete(CONTEXT, owner, "published") + finally: + event.remove(sqlite_session_factory, "before_commit", reject) + with sqlite_session_factory() as session: + assert session.get(Workflow, "published") is not None + assert session.get(WorkflowAgentNodeBinding, "binding") is not None + agent = session.get(Agent, "inline") + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + assert dispatched == [] + else: + workflow_application.console_workflows.delete(CONTEXT, owner, "published") + assert dispatched == [{"inline"}] + + +@pytest.mark.parametrize("case", ["active", "draft", "tool", "missing", "foreign", "kind"]) +def test_owner_delete_preserves_version_constraints( + owner: WorkflowOwner, + sqlite_session_factory: sessionmaker[Session], + workflow_application: ApplicationServices, + case: str, +) -> None: + from models.tools import WorkflowToolProvider + from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError + + if case == "kind" and owner.kind != "snippet": + return + with sqlite_session_factory.begin() as session: + workflow = make_workflow(workflow_id="target", app_id=owner.id, version="draft" if case == "draft" else "v1") + workflow.kind = WorkflowKind.SNIPPET if owner.kind == "snippet" and case != "kind" else WorkflowKind.STANDARD + if case == "foreign": + workflow.tenant_id = "other" + session.add(workflow) + if case == "active": + row = session.get(CustomizedSnippet if owner.kind == "snippet" else Pipeline, owner.id) + assert row is not None + row.workflow_id = workflow.id + if case == "tool": + session.add( + WorkflowToolProvider( + tenant_id="tenant-1", + app_id=owner.id, + version="v1", + name="Tool", + label="Tool", + icon="", + description="", + parameter_configuration="[]", + privacy_policy="", + user_id="account-1", + ) + ) + expected = ( + WorkflowInUseError + if case in {"active", "tool"} + else DraftWorkflowDeletionError + if case == "draft" + else WorkflowNotFoundError + ) + with pytest.raises(expected): + workflow_application.console_workflows.delete(CONTEXT, owner, "missing" if case == "missing" else "target") + with sqlite_session_factory() as session: + assert session.get(Workflow, "target") is not None diff --git a/api/tests/unit_tests/repositories/workflow/test_pipeline_publication.py b/api/tests/unit_tests/repositories/workflow/test_pipeline_publication.py index 34e9eed5536ebc..9bb66d80b2614f 100644 --- a/api/tests/unit_tests/repositories/workflow/test_pipeline_publication.py +++ b/api/tests/unit_tests/repositories/workflow/test_pipeline_publication.py @@ -309,7 +309,10 @@ def test_invalid_llm_environment_reference_is_rejected( ] } ) - with pytest.raises(ValueError, match="LLM environment variable 'missing' was not found"): + with pytest.raises( + ValueError, + match="LLM environment variable 'missing' was not found or is not an LLM variable", + ): service.publish(CONTEXT, "pipeline-1") assert indexes.dispatched == [] diff --git a/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py b/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py index 9827b5d338536a..8464ad7416daf3 100644 --- a/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py +++ b/api/tests/unit_tests/services/agent/chat/test_tool_invocation.py @@ -21,11 +21,11 @@ ToolInvokeMeta, ToolProviderType, ) +from extensions.application_services.workflow import build_workflow_execution_dependencies from graphon.model_runtime.entities import PromptMessage from models.model import Message from services.agent.chat.cot_runner import CotAgentRunner from services.agent.chat.ports import AgentToolInvoker -from services.app.generation.adapters.agent_chat import _SessionBoundAgentToolInvoker from services.app.generation.ports import AgentMessageRecords, MessageFileWriter from tests.helpers.unit.model_factories import make_message @@ -98,7 +98,7 @@ def test_session_bound_invoker_consumes_lazy_messages_before_closing_session( fail_after_first_message: bool, ) -> None: tool_sessions: sessionmaker[Session] = sessionmaker(class_=_TrackingSession) - invoker = _SessionBoundAgentToolInvoker(tool_sessions) + invoker = build_workflow_execution_dependencies(tool_sessions).agent_tool_invoker tool = _LazyTool(fail_after_first_message=fail_after_first_message) text, files, meta = invoker( diff --git a/api/tests/unit_tests/services/agent/test_deletion_service.py b/api/tests/unit_tests/services/agent/test_deletion_service.py index 26e9c472a6b3b9..44dcb65d65a641 100644 --- a/api/tests/unit_tests/services/agent/test_deletion_service.py +++ b/api/tests/unit_tests/services/agent/test_deletion_service.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.sql.dml import Delete +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigDraft, @@ -27,7 +28,6 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig diff --git a/api/tests/unit_tests/services/agent/test_dsl_service.py b/api/tests/unit_tests/services/agent/test_dsl_service.py index 680f4fd7569d8d..1e5f54a27baa5e 100644 --- a/api/tests/unit_tests/services/agent/test_dsl_service.py +++ b/api/tests/unit_tests/services/agent/test_dsl_service.py @@ -1,4 +1,4 @@ -import json +from typing import TypedDict from unittest.mock import Mock from uuid import uuid4 @@ -7,6 +7,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from enums.agent import WorkflowAgentBindingType from graphon.enums import BuiltinNodeTypes from models.account import Account from models.agent import ( @@ -20,23 +21,23 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) -from models.agent_config_entities import AgentConfigFileRefConfig, AgentConfigSkillRefConfig, AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.dataset import Dataset from models.enums import AppStatus from models.model import App, AppMode, IconType from models.tools import ToolFile from models.workflow import Workflow, WorkflowType +from repositories.agent.creation_repository import WorkflowAgentCreationRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope from services.agent.dsl_entities import ( AGENT_NODE_JOB_DSL_KEY, AGENT_PACKAGE_REF_KEY, AgentPackage, - AgentPackageMetadata, make_portable_agent_package, ) -from services.agent.dsl_service import AgentDslService, AgentPackageImportResult, is_agent_v2_graph +from services.agent.dsl_service import AgentDslService, AgentPackageImportResult from services.entities.dsl_entities import DslImportWarning from tests.helpers.unit.model_factories import make_upload_file @@ -120,8 +121,18 @@ def _workflow(*, workflow_id: str = "workflow-1") -> Workflow: ) -def _agent_node(node_id: str, binding: object | None = None) -> dict: - data = {"type": BuiltinNodeTypes.AGENT, "version": "2", "agent_node_kind": "dify_agent"} +class _AgentNode(TypedDict): + id: str + data: dict[str, object] + + +class _Graph(TypedDict): + nodes: list[_AgentNode] + edges: list[object] + + +def _agent_node(node_id: str, binding: object | None = None) -> _AgentNode: + data: dict[str, object] = {"type": BuiltinNodeTypes.AGENT, "version": "2", "agent_node_kind": "dify_agent"} if binding is not None: data["agent_binding"] = binding return {"id": node_id, "data": data} @@ -260,7 +271,7 @@ def test_agent_package_normalizes_legacy_null_missing_asset_file_ids() -> None: {"name": "guide.md", "file_kind": "tool_file", "file_id": None, "is_missing": False}, ], ) -def test_agent_package_rejects_null_file_id_for_available_assets(asset: dict) -> None: +def test_agent_package_rejects_null_file_id_for_available_assets(asset: dict[str, object]) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()).model_dump(mode="json") target = "config_files" if "file_kind" in asset else "config_skills" package["soul"][target] = [asset] @@ -344,7 +355,7 @@ def test_export_agent_app_uses_draft_or_active_snapshot(sqlite_session: Session, def test_export_workflow_packages_deduplicates_shared_agent(sqlite_session: Session) -> None: - graph = {"nodes": [_agent_node("node-1"), _agent_node("node-2")], "edges": []} + graph: _Graph = {"nodes": [_agent_node("node-1"), _agent_node("node-2")], "edges": []} bindings = [ WorkflowAgentNodeBinding( id=f"binding-{node_id}", @@ -387,7 +398,7 @@ def test_export_workflow_packages_rejects_incomplete_binding(sqlite_session: Ses def test_graph_without_package_bindings_removes_portable_fields() -> None: - graph = { + graph: _Graph = { "nodes": [ _agent_node( "portable", @@ -466,7 +477,7 @@ def create_backing_agent(**_kwargs: object) -> Agent: def test_import_workflow_packages_materializes_every_package_binding_as_inline(sqlite_session: Session) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()) - graph = { + graph: _Graph = { "nodes": [ _agent_node( "roster-1", @@ -522,7 +533,7 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s workflow = _workflow() result, warnings, retirement_candidates = service.import_workflow_packages( - workflow=workflow, + workflow=workflow_binding_scope(workflow), portable_graph=graph, raw_packages={"agent_1": package.model_dump(mode="json")}, account=_account(), @@ -545,7 +556,6 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s ] assert all(binding["binding_type"] == WorkflowAgentBindingType.INLINE_AGENT.value for binding in bindings) assert AGENT_NODE_JOB_DSL_KEY not in result["nodes"][0]["data"] - assert json.loads(workflow.graph) == result added_bindings = sqlite_session.scalars( select(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.workflow_id == workflow.id) ).all() @@ -564,59 +574,19 @@ def test_import_workflow_packages_materializes_every_package_binding_as_inline(s ], ) def test_import_workflow_packages_rejects_invalid_package_binding( - sqlite_session: Session, binding: dict, error: str + sqlite_session: Session, binding: dict[str, object], error: str ) -> None: package = make_portable_agent_package(_agent(), AgentSoulConfig()) with pytest.raises(ValueError, match=error): AgentDslService(sqlite_session).import_workflow_packages( - workflow=_workflow(), + workflow=workflow_binding_scope(_workflow()), portable_graph={"nodes": [_agent_node("node-1", binding)], "edges": []}, raw_packages={"agent_1": package.model_dump(mode="json")}, account=_account(), ) -def test_clone_inline_binding_copies_soul(unbound_session: Session) -> None: - service = AgentDslService(unbound_session) - target_agent = _agent( - agent_id="target-agent", - scope=AgentScope.WORKFLOW_ONLY, - source=AgentSource.WORKFLOW, - ) - target_snapshot = _snapshot(snapshot_id="target-snapshot", agent_id=target_agent.id) - service._create_workflow_only_agent = Mock(return_value=(target_agent, target_snapshot)) - source_agent = _agent() - source_soul = AgentSoulConfig( - config_note="source", - config_skills=[AgentConfigSkillRefConfig(name="summarizer", file_id="skill-file-1")], - config_files=[AgentConfigFileRefConfig(name="brief.pdf", file_kind="upload_file", file_id="config-file-1")], - ) - source_snapshot = _snapshot(soul=source_soul) - workflow = _workflow() - - result = service.clone_inline_binding_for_node( - workflow=workflow, - node_id="target-node", - source_agent=source_agent, - source_snapshot=source_snapshot, - account_id="account-1", - ) - - assert result == (target_agent, target_snapshot) - create_kwargs = service._create_workflow_only_agent.call_args.kwargs - assert create_kwargs["metadata"].name == source_agent.name - cloned_soul = create_kwargs["soul"] - assert cloned_soul.config_note == "source" - assert [(item.name, item.file_kind, item.file_id) for item in cloned_soul.config_skills] == [ - ("summarizer", "tool_file", "skill-file-1") - ] - assert [(item.name, item.file_kind, item.file_id) for item in cloned_soul.config_files] == [ - ("brief.pdf", "upload_file", "config-file-1") - ] - assert create_kwargs["source"] == AgentSource.WORKFLOW - - def test_extract_package_dependencies_covers_models_features_tools_and_knowledge( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: @@ -793,7 +763,7 @@ def test_create_imported_inline_agent_uses_import_provenance(unbound_session: Se workflow = _workflow() result = service._create_imported_inline_agent( - workflow=workflow, + workflow=workflow_binding_scope(workflow), node_id="node-1", account=_account(), package=make_portable_agent_package(_agent(), soul), @@ -811,16 +781,11 @@ def test_create_workflow_only_agent_sets_backing_app_and_snapshot( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: service = AgentDslService(sqlite_session) - roster_service = Mock() - backing_app = _app() - backing_app.id = "backing-app" - roster_service.create_hidden_backing_app_for_workflow_agent.return_value = backing_app - monkeypatch.setattr("services.agent.dsl_service.AgentRosterService", Mock(return_value=roster_service)) - monkeypatch.setattr("services.agent.dsl_service.agent_soul_has_model", Mock(return_value=True)) + monkeypatch.setattr("repositories.agent.creation_repository.agent_soul_has_model", Mock(return_value=True)) workflow = _workflow() agent, snapshot = service._create_workflow_only_agent( - workflow=workflow, + workflow=workflow_binding_scope(workflow), node_id="node-1", account_id="account-1", metadata=AgentPackageMetadata(name="Inline", icon_type=AgentIconType.EMOJI.value), @@ -832,7 +797,7 @@ def test_create_workflow_only_agent_sets_backing_app_and_snapshot( assert sqlite_session.get(Agent, agent.id) is agent assert sqlite_session.get(AgentConfigSnapshot, snapshot.id) is snapshot assert agent.scope == AgentScope.WORKFLOW_ONLY - assert agent.backing_app_id == "backing-app" + assert sqlite_session.get(App, agent.backing_app_id) is not None assert agent.active_config_snapshot_id == snapshot.id assert agent.active_config_has_model is True assert agent.active_config_is_published is True @@ -976,9 +941,8 @@ def test_create_snapshot_increments_version_and_records_revision(sqlite_session: ] ) sqlite_session.commit() - service = AgentDslService(sqlite_session) - snapshot = service._create_snapshot( + snapshot = WorkflowAgentCreationRepository(sqlite_session).create_snapshot( tenant_id="tenant-1", agent=agent, account_id="account-1", @@ -1011,7 +975,7 @@ def test_unique_roster_name_uses_first_available_suffix(sqlite_session: Session) assert result == "Agent import 2" -def test_require_helpers_and_graph_detection(sqlite_session: Session) -> None: +def test_require_helpers(sqlite_session: Session) -> None: agent = _agent() snapshot = _snapshot() sqlite_session.add_all([agent, snapshot]) @@ -1029,5 +993,3 @@ def test_require_helpers_and_graph_detection(sqlite_session: Session) -> None: assert AgentDslService._agent_icon_type(AgentIconType.EMOJI.value) == AgentIconType.EMOJI assert AgentDslService._agent_icon_type(None) is None - assert is_agent_v2_graph({"nodes": [_agent_node("agent")]}) is True - assert is_agent_v2_graph({"nodes": ["invalid", {"data": {"type": "start"}}]}) is False diff --git a/api/tests/unit_tests/services/agent/test_knowledge_spaces.py b/api/tests/unit_tests/services/agent/test_knowledge_spaces.py new file mode 100644 index 00000000000000..e2d65fc78f7de2 --- /dev/null +++ b/api/tests/unit_tests/services/agent/test_knowledge_spaces.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from unittest.mock import Mock + +import pytest +from sqlalchemy.orm import Session, sessionmaker + +from models.workflow import Workflow +from services.agent import knowledge_spaces + + +@dataclass +class _RecordingAppBindings: + calls: list[dict[str, object]] = field(default_factory=list) + + def sync_workflow_bindings(self, **kwargs: object) -> list[object]: + self.calls.append(kwargs) + return [] + + +@dataclass +class _Runtime: + app_bindings: _RecordingAppBindings + + +def _workflow(nodes: list[dict[str, object]]) -> Workflow: + return Workflow( + id="published-workflow", + tenant_id="tenant-1", + app_id="app-1", + version="2026-10-26T00:00:00Z", + graph=json.dumps({"nodes": nodes, "edges": []}), + created_by="account-1", + ) + + +def _patch_runtime(monkeypatch: pytest.MonkeyPatch, bindings: _RecordingAppBindings) -> sessionmaker[Session]: + session_maker = Mock(spec=sessionmaker) + monkeypatch.setattr( + "services.knowledge_fs.runtime.get_knowledge_fs_runtime", + lambda actual_session_maker: _Runtime(bindings) if actual_session_maker is session_maker else None, + ) + return session_maker + + +def test_sync_workflow_bindings_unions_direct_and_frozen_agent_spaces( + monkeypatch: pytest.MonkeyPatch, +) -> None: + flush = Mock() + session = Mock(spec=Session, flush=flush) + bindings = _RecordingAppBindings() + session_maker = _patch_runtime(monkeypatch, bindings) + owning_session = session + + def collect_agent_spaces(*, session: Session, workflow: Workflow) -> tuple[str, ...]: + assert session is owning_session + assert flush.call_count == 1 + assert workflow.id == "published-workflow" + return ("space-b", "space-c") + + monkeypatch.setattr(knowledge_spaces, "collect_workflow_agent_knowledge_space_ids", collect_agent_spaces) + workflow = _workflow( + [ + { + "id": "knowledge", + "data": { + "type": "knowledge-retrieval-v2", + "control_space_ids": ["space-a", "space-b"], + "query_variable_selector": ["start", "query"], + }, + }, + { + "id": "agent", + "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}, + }, + ] + ) + + knowledge_spaces.sync_workflow_app_knowledge_bindings( + session=session, + session_maker=session_maker, + workflow=workflow, + ) + + assert bindings.calls == [ + { + "tenant_id": "tenant-1", + "actor_account_id": "account-1", + "app_id": "app-1", + "control_space_ids": ["space-a", "space-b", "space-c"], + "session": session, + } + ] + + +def test_sync_workflow_bindings_clears_stale_grants(monkeypatch: pytest.MonkeyPatch) -> None: + session = Mock(spec=Session) + session.scalar.return_value = "existing-join" + bindings = _RecordingAppBindings() + session_maker = _patch_runtime(monkeypatch, bindings) + + knowledge_spaces.sync_workflow_app_knowledge_bindings( + session=session, + session_maker=session_maker, + workflow=_workflow([]), + ) + + assert bindings.calls == [ + { + "tenant_id": "tenant-1", + "actor_account_id": "account-1", + "app_id": "app-1", + "control_space_ids": [], + "session": session, + } + ] + + +def test_sync_workflow_bindings_skips_empty_graph_without_stale_grants(monkeypatch: pytest.MonkeyPatch) -> None: + session = Mock(spec=Session) + session.scalar.return_value = None + session_maker = Mock(spec=sessionmaker) + monkeypatch.setattr( + "services.knowledge_fs.runtime.get_knowledge_fs_runtime", + Mock(side_effect=AssertionError("runtime must not be constructed for a no-op")), + ) + + knowledge_spaces.sync_workflow_app_knowledge_bindings( + session=session, + session_maker=session_maker, + workflow=_workflow([]), + ) + + session.scalar.assert_called_once() diff --git a/api/tests/unit_tests/services/agent/test_log_service.py b/api/tests/unit_tests/services/agent/test_log_service.py index e34495bc3c66ef..c4da94a2320c93 100644 --- a/api/tests/unit_tests/services/agent/test_log_service.py +++ b/api/tests/unit_tests/services/agent/test_log_service.py @@ -36,6 +36,7 @@ from models.tools import ApiToolProvider, ToolFile from services.agent.log_contracts import AgentLogAppNotFoundError, AgentLogConfigurationError, AgentLogNotFoundError from services.agent.log_service import AgentLogService +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.model_factories import make_account, make_app, make_conversation, make_message, make_upload_file CONTEXT = RequestContext("request", None, "viewer", "tenant") @@ -86,7 +87,7 @@ def _provide_app_context() -> None: @pytest.fixture -def store(monkeypatch: pytest.MonkeyPatch) -> Iterator[AgentLogStore]: +def store(monkeypatch: pytest.MonkeyPatch, workflow_variables: WorkflowExecutionVariables) -> Iterator[AgentLogStore]: engine = create_engine("sqlite://", poolclass=QueuePool) tables: list[Table] = [] for model in ( @@ -196,7 +197,7 @@ def reject_global(*_args: object, **_kwargs: object) -> None: message_files='["generated-file"]', ) ) - service = build_agent_app_services(database_client=sessions).logs + service = build_agent_app_services(database_client=sessions, variables=workflow_variables).logs try: yield sessions, service finally: diff --git a/api/tests/unit_tests/services/agent/test_observability_service.py b/api/tests/unit_tests/services/agent/test_observability_service.py index 1bfa9692b94408..7822ec1fbfa90b 100644 --- a/api/tests/unit_tests/services/agent/test_observability_service.py +++ b/api/tests/unit_tests/services/agent/test_observability_service.py @@ -9,9 +9,10 @@ from core.app.entities.app_invoke_entities import InvokeFrom as _agent_observability_service_InvokeFrom from core.workflow.node_execution_process_data import WORKFLOW_AGENT_BINDING_ID_KEY +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowNodeExecutionStatus from libs.datetime_utils import naive_utc_now -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding +from models.agent import WorkflowAgentNodeBinding from models.agent_config_entities import WorkflowNodeJobConfig from models.enums import ( ConversationFromSource, diff --git a/api/tests/unit_tests/services/agent/test_retirement_service.py b/api/tests/unit_tests/services/agent/test_retirement_service.py index 471dc4344d3205..e8e4dc3315374c 100644 --- a/api/tests/unit_tests/services/agent/test_retirement_service.py +++ b/api/tests/unit_tests/services/agent/test_retirement_service.py @@ -1,10 +1,9 @@ -from contextlib import nullcontext -from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, create_autospec import pytest -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker +from enums.agent import WorkflowAgentBindingType from models.agent import ( Agent, AgentConfigVersionKind, @@ -17,37 +16,25 @@ AgentWorkspace, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.enums import AppStatus from models.model import App, AppMode from models.workflow import Workflow, WorkflowType -from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.agent.workspace_service import AgentWorkspaceService +from repositories.agent.retirement_repository import AgentRetirement, WorkflowAgentRetirementRepository +from services.agent.retirement_service import WorkflowAgentRetirementService -def test_retire_unowned_failure_propagates(monkeypatch: pytest.MonkeyPatch) -> None: - context = MagicMock() +def test_retire_unowned_failure_propagates() -> None: + repository = create_autospec(WorkflowAgentRetirementRepository, instance=True, spec_set=True) error = RuntimeError("retirement failed") - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: context, - ) - monkeypatch.setattr( - WorkflowAgentRetirementService, - "archive_unowned", - MagicMock(side_effect=error), - ) - + repository.retire.side_effect = error with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(repository).retire_unowned( tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1", ) - assert exc_info.value is error @@ -81,6 +68,7 @@ def _workflow_only_agent(*, backing_app_id: str | None = None) -> Agent: def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], workflow_version: str, pointer_to_owner: bool, mismatched_key: str | None, @@ -135,18 +123,15 @@ def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( ) sqlite_session.add_all([agent, app, workflow, binding]) sqlite_session.commit() - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: nullcontext(sqlite_session), - ) celery_delay = MagicMock() monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", celery_delay) - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sqlite_session_factory)).retire_unowned( tenant_id="tenant-1", agent_ids=[agent.id], account_id="account-1", ) + sqlite_session.expire_all() stored_agent = sqlite_session.get(Agent, agent.id) assert stored_agent is not None assert stored_agent.status is expected_status @@ -164,6 +149,7 @@ def test_retire_unowned_requires_an_exact_persisted_workflow_owner_key( def test_retire_unowned_archives_orphan_and_retires_resources( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], ) -> None: agent = _workflow_only_agent(backing_app_id="hidden-app-1") hidden_app = App( @@ -207,24 +193,23 @@ def test_retire_unowned_archives_orphan_and_retires_resources( ) sqlite_session.add_all([agent, hidden_app, home, workspace, binding]) sqlite_session.commit() - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: nullcontext(sqlite_session), - ) + hidden_app_id = hidden_app.id + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) cleanup_app = MagicMock() enqueue_collection = MagicMock() - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) + monkeypatch.setattr("services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_app) monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(WorkflowAgentRetirementRepository(sqlite_session_factory)).retire_unowned( tenant_id="tenant-1", agent_ids=[agent.id], account_id="account-1", ) + sqlite_session.expire_all() stored_agent = sqlite_session.get(Agent, agent.id) stored_binding = sqlite_session.get(AgentWorkspaceBinding, binding.id) stored_workspace = sqlite_session.get(AgentWorkspace, workspace.id) @@ -233,12 +218,12 @@ def test_retire_unowned_archives_orphan_and_retires_resources( assert stored_binding is not None assert stored_workspace is not None assert stored_home is not None - assert sqlite_session.get(App, hidden_app.id) is None + assert sqlite_session.get(App, hidden_app_id) is None assert stored_agent.status is AgentStatus.ARCHIVED assert stored_binding.status is AgentWorkingResourceStatus.RETIRED assert stored_workspace.status is AgentWorkingResourceStatus.RETIRED assert stored_home.status is AgentWorkingResourceStatus.RETIRED - cleanup_app.assert_called_once_with(tenant_id="tenant-1", app_id=hidden_app.id) + cleanup_app.assert_called_once_with(tenant_id="tenant-1", app_id=hidden_app_id) enqueue_collection.assert_called_once_with( tenant_id="tenant-1", workspace_ids=[workspace.id], @@ -249,44 +234,21 @@ def test_retire_unowned_archives_orphan_and_retires_resources( def test_hidden_app_enqueue_failure_prevents_agent_purge_enqueue(monkeypatch: pytest.MonkeyPatch) -> None: - context = MagicMock() - session = context.__enter__.return_value - session.scalars.side_effect = [ - SimpleNamespace( - all=MagicMock( - return_value=[ - SimpleNamespace(backing_app_id="hidden-app-1"), - SimpleNamespace(backing_app_id="hidden-app-2"), - ] - ) - ), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - SimpleNamespace(all=MagicMock(return_value=[])), - ] - monkeypatch.setattr( - "services.agent.legacy_retirement_service.session_factory.create_session", - lambda: context, + repository = create_autospec(WorkflowAgentRetirementRepository, instance=True, spec_set=True) + repository.retire.return_value = AgentRetirement( + ["agent-1", "agent-2"], ["hidden-app-1", "hidden-app-2"], [], [], [] ) - monkeypatch.setattr( - WorkflowAgentRetirementService, - "archive_unowned", - MagicMock(return_value=["agent-1", "agent-2"]), - ) - monkeypatch.setattr(AgentWorkspaceService, "retire_all_for_app", MagicMock(return_value=[])) - monkeypatch.setattr(AgentHomeSnapshotService, "retire_all_for_agent", MagicMock(return_value=[])) error = RuntimeError("broker unavailable") cleanup_app = MagicMock(side_effect=[None, error]) - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) + monkeypatch.setattr("services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_app) enqueue_collection = MagicMock() monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( + WorkflowAgentRetirementService(repository).retire_unowned( tenant_id="tenant-1", agent_ids=["agent-1", "agent-2"], account_id="account-1", @@ -295,103 +257,3 @@ def test_hidden_app_enqueue_failure_prevents_agent_purge_enqueue(monkeypatch: py assert exc_info.value is error assert [call.kwargs["app_id"] for call in cleanup_app.call_args_list] == ["hidden-app-1", "hidden-app-2"] enqueue_collection.assert_not_called() - - -def test_retire_unowned_retry_after_hidden_app_enqueue_failure_preserves_full_collector_payload( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - agent = _workflow_only_agent(backing_app_id="hidden-app-1") - hidden_app = App( - id="hidden-app-1", - tenant_id="tenant-1", - name="Inline Agent runtime", - mode=AppMode.AGENT, - status=AppStatus.NORMAL, - enable_site=False, - enable_api=False, - ) - home = AgentHomeSnapshot( - id="home-1", - tenant_id="tenant-1", - agent_id=agent.id, - snapshot_ref="home-ref", - status=AgentWorkingResourceStatus.ACTIVE, - ) - workspace = AgentWorkspace( - id="workspace-1", - tenant_id="tenant-1", - app_id=hidden_app.id, - owner_type=AgentWorkspaceOwnerType.CONVERSATION, - owner_id="conversation-1", - owner_scope_key="root", - backend_workspace_ref="workspace-ref", - status=AgentWorkingResourceStatus.ACTIVE, - active_guard=1, - ) - binding = AgentWorkspaceBinding( - id="binding-1", - tenant_id="tenant-1", - app_id=hidden_app.id, - workspace_id=workspace.id, - agent_id=agent.id, - base_home_snapshot_id=home.id, - agent_config_version_id="config-1", - agent_config_version_kind=AgentConfigVersionKind.SNAPSHOT, - backend_binding_ref="binding-ref", - status=AgentWorkingResourceStatus.ACTIVE, - ) - sqlite_session.add_all([agent, hidden_app, home, workspace, binding]) - sqlite_session.commit() - agent_id = agent.id - hidden_app_id = hidden_app.id - home_id = home.id - workspace_id = workspace.id - binding_id = binding.id - error = RuntimeError("broker unavailable") - cleanup_app = MagicMock(side_effect=[error, None]) - enqueue_collection = MagicMock() - monkeypatch.setattr("services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", cleanup_app) - monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", - enqueue_collection, - ) - - with pytest.raises(RuntimeError) as exc_info: - WorkflowAgentRetirementService.retire_unowned( - tenant_id="tenant-1", - agent_ids=[agent_id], - account_id="account-1", - ) - - assert exc_info.value is error - sqlite_session.expire_all() - stored_agent = sqlite_session.get(Agent, agent_id) - stored_workspace = sqlite_session.get(AgentWorkspace, workspace_id) - stored_binding = sqlite_session.get(AgentWorkspaceBinding, binding_id) - stored_home = sqlite_session.get(AgentHomeSnapshot, home_id) - assert stored_agent is not None - assert stored_workspace is not None - assert stored_binding is not None - assert stored_home is not None - assert stored_agent.status is AgentStatus.ARCHIVED - assert sqlite_session.get(App, hidden_app_id) is None - assert stored_workspace.status is AgentWorkingResourceStatus.RETIRED - assert stored_binding.status is AgentWorkingResourceStatus.RETIRED - assert stored_home.status is AgentWorkingResourceStatus.RETIRED - enqueue_collection.assert_not_called() - - WorkflowAgentRetirementService.retire_unowned( - tenant_id="tenant-1", - agent_ids=[agent_id], - account_id="account-1", - ) - - assert cleanup_app.call_count == 2 - enqueue_collection.assert_called_once_with( - tenant_id="tenant-1", - workspace_ids=[workspace_id], - binding_ids=[binding_id], - home_snapshot_ids=[home_id], - purge_agent_ids=[agent_id], - ) diff --git a/api/tests/unit_tests/services/agent/test_roster_package_importer.py b/api/tests/unit_tests/services/agent/test_roster_package_importer.py index 9a8a1ce7c564cb..ad1ca969aa8ffb 100644 --- a/api/tests/unit_tests/services/agent/test_roster_package_importer.py +++ b/api/tests/unit_tests/services/agent/test_roster_package_importer.py @@ -31,17 +31,12 @@ AgentSource, AgentStatus, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.base import Base, TypeBase from models.model import App, AppMode, AppModelConfig, InstalledApp, Site, UploadFile from models.tools import ToolFile from services.agent import roster_package_importer as importer_module -from services.agent.dsl_entities import ( - AgentPackage, - AgentPackageMetadata, - AgentPackageWorkspaceSkill, - make_agent_app_dsl, -) +from services.agent.dsl_entities import AgentPackage, AgentPackageWorkspaceSkill, make_agent_app_dsl from services.agent.errors import ( AgentNameConflictError, InvalidRosterAgentPackageError, diff --git a/api/tests/unit_tests/services/agent/test_roster_package_reader.py b/api/tests/unit_tests/services/agent/test_roster_package_reader.py index 4f737273a5ff5f..0d525eb95fd43c 100644 --- a/api/tests/unit_tests/services/agent/test_roster_package_reader.py +++ b/api/tests/unit_tests/services/agent/test_roster_package_reader.py @@ -30,13 +30,13 @@ AgentSource, AgentStatus, ) -from models.agent_config_entities import AgentSoulConfig +from models.agent_config_entities import AgentPackageMetadata, AgentSoulConfig from models.enums import AppStatus, CreatorUserRole, CustomizeTokenStrategy from models.model import App, AppMode, IconType, Site from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest from models.tools import ToolFile from services.agent import roster_package_exporter as roster_package_exporter_module -from services.agent.dsl_entities import AgentAppDsl, AgentPackage, AgentPackageMetadata, make_agent_app_dsl +from services.agent.dsl_entities import AgentAppDsl, AgentPackage, make_agent_app_dsl from services.agent.errors import ( AgentVersionNotFoundError, InvalidRosterAgentPackageError, diff --git a/api/tests/unit_tests/services/agent/test_roster_service.py b/api/tests/unit_tests/services/agent/test_roster_service.py index 067f92f7cf09ae..1221bdd254d611 100644 --- a/api/tests/unit_tests/services/agent/test_roster_service.py +++ b/api/tests/unit_tests/services/agent/test_roster_service.py @@ -8,9 +8,11 @@ from pytest_mock import MockerFixture from sqlalchemy import event, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker -from core.workflow.nodes.agent_v2.validators import WorkflowAgentNodeValidationError +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.agent_bindings import build_workflow_agent_service +from extensions.application_services.workflow_variables import build_workflow_variable_service from models.account import Account from models.agent import ( Agent, @@ -28,7 +30,6 @@ AgentStatus, AgentWorkspaceBinding, AgentWorkspaceOwnerType, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import ( @@ -37,14 +38,17 @@ DeclaredOutputConfig, DeclaredOutputType, WorkflowNodeJobConfig, + agent_soul_has_model, ) from models.enums import ConversationFromSource, ConversationStatus from models.model import App, AppMode, AppModelConfig, Conversation, IconType, Message from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest from models.tools import ToolFile from models.workflow import Workflow +from repositories.agent.config_repository import AgentConfigRepository +from repositories.agent.workflow_binding_repository import workflow_binding_scope +from repositories.agent_workspace_repository import AgentWorkspaceRepository from services.agent import composer_service, roster_service -from services.agent.agent_soul_state import agent_soul_has_model from services.agent.composer_service import AgentComposerService from services.agent.composer_validator import ComposerConfigValidator from services.agent.errors import ( @@ -57,8 +61,10 @@ InvalidComposerConfigError, ) from services.agent.home_snapshot_service import AgentHomeSnapshotService -from services.agent.legacy_workflow_publish_service import WorkflowAgentPublishService from services.agent.roster_service import AgentRosterService +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.agent.workflow_resources_gateway import WorkflowAgentSkillReader +from services.agent.workflow_validator import WorkflowAgentNodeValidationError from services.agent.workspace_service import AgentWorkspaceService from services.app_service import AppListParams, AppService from services.entities.agent_entities import ( @@ -67,6 +73,7 @@ ComposerSaveStrategy, ComposerVariant, ) +from services.skill_management_service import SkillManagementService from tests.helpers.unit.model_factories import make_account, make_app, make_conversation, make_workflow @@ -271,13 +278,13 @@ def test_load_workflow_composer_serializes_existing_binding(monkeypatch: pytest. monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **kwargs: _agent(), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **kwargs: _snapshot(snapshot_id="version-1"), ) monkeypatch.setattr( @@ -311,10 +318,10 @@ def test_load_workflow_composer_uses_roster_preview_snapshot(monkeypatch: pytest monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: agent) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: agent) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: _snapshot(snapshot_id=kwargs["version_id"]), ) monkeypatch.setattr( @@ -364,10 +371,10 @@ def test_load_workflow_composer_uses_inline_preview_snapshot(monkeypatch: pytest monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: agent) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: agent) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: _snapshot(snapshot_id=kwargs["version_id"]), ) monkeypatch.setattr( @@ -537,13 +544,13 @@ def test_save_workflow_composer_dispatches_save_strategy(monkeypatch, strategy, monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_lock_workflow_binding", lambda **kwargs: None) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **kwargs: _agent(), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **kwargs: _snapshot(snapshot_id="version-1"), ) @@ -592,8 +599,8 @@ def test_save_workflow_composer_commits_before_retiring_replaced_inline_agent( workflow_id="workflow-1", workflow_version="draft", node_id="node-1", - node_job_config=WorkflowNodeJobConfig(), agent_id="old-inline-agent", + node_job_config=WorkflowNodeJobConfig(), binding_type=WorkflowAgentBindingType.INLINE_AGENT, ) new_binding = WorkflowAgentNodeBinding( @@ -608,16 +615,17 @@ def test_save_workflow_composer_commits_before_retiring_replaced_inline_agent( current_snapshot_id="version-1", ) monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **_kwargs: _workflow()) - monkeypatch.setattr(AgentComposerService, "_lock_workflow_binding", lambda **_kwargs: old_binding) + session.add(old_binding) + session.commit() monkeypatch.setattr(AgentComposerService, "_save_as_new_agent", lambda **_kwargs: new_binding) monkeypatch.setattr( - AgentComposerService, - "_get_agent_if_present", + AgentConfigRepository, + "get_agent", lambda **_kwargs: _agent(agent_id="new-agent"), ) monkeypatch.setattr( - AgentComposerService, - "_get_version_if_present", + AgentConfigRepository, + "get_snapshot", lambda **_kwargs: _snapshot(snapshot_id="version-1"), ) monkeypatch.setattr(AgentComposerService, "_serialize_workflow_state", lambda **_kwargs: {"state": "ok"}) @@ -629,7 +637,7 @@ def retire_unowned(**kwargs): assert kwargs["agent_ids"] == {"old-inline-agent"} events.append("retire") - monkeypatch.setattr(composer_service.WorkflowAgentRetirementService, "retire_unowned", retire_unowned) + monkeypatch.setattr(composer_service.WorkflowAgentRetirementService, "retire_unowned", staticmethod(retire_unowned)) payload = ComposerSavePayload.model_validate( { "variant": ComposerVariant.WORKFLOW, @@ -770,8 +778,8 @@ def test_load_agent_app_composer_exposes_draft_save_only(monkeypatch: pytest.Mon ) monkeypatch.setattr(AgentComposerService, "_require_agent_app_agent", lambda **kwargs: agent) - monkeypatch.setattr(AgentComposerService, "_get_agent_draft", lambda **kwargs: draft) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: None) + monkeypatch.setattr(AgentConfigRepository, "get_draft", lambda **kwargs: draft) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: None) monkeypatch.setattr(AgentComposerService, "_serialize_agent", lambda _agent: {"id": _agent.id}) monkeypatch.setattr(AgentComposerService, "_serialize_version", lambda _version: None) monkeypatch.setattr(AgentComposerService, "_serialize_draft", lambda _draft: {"id": "draft-1"}) @@ -962,7 +970,7 @@ def test_save_agent_app_composer_updates_normal_draft(monkeypatch: pytest.Monkey ) ), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr( AgentComposerService, "load_agent_composer", @@ -1026,7 +1034,7 @@ def test_save_agent_app_composer_keeps_published_when_draft_matches_active_snaps home_snapshot_id="home-initial", ), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr( AgentComposerService, "load_agent_composer", @@ -1699,7 +1707,7 @@ def test_build_apply_retires_normal_preview_binding_before_replacing_draft_home( validate_generation = MagicMock() enqueue_collection = MagicMock(side_effect=lambda **_kwargs: lifecycle.append("enqueue")) monkeypatch.setattr(AgentWorkspaceService, "get_active_binding", get_active_binding) - monkeypatch.setattr(AgentWorkspaceService, "validate_binding_generation", validate_generation) + monkeypatch.setattr(AgentWorkspaceRepository, "validate_binding_generation", validate_generation) monkeypatch.setattr(AgentWorkspaceService, "retire_binding", retire_binding) monkeypatch.setattr(composer_service, "enqueue_agent_resource_collection", enqueue_collection) @@ -2421,7 +2429,9 @@ def test_agent_app_build_draft_apply_marks_unpublished_when_build_draft_differs( ) -def test_agent_app_composer_candidates_and_impact(monkeypatch: pytest.MonkeyPatch, sqlite_session: Session): +def test_agent_app_composer_candidates_and_impact( + monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers +): session = sqlite_session bindings = [ WorkflowAgentNodeBinding( @@ -2450,14 +2460,16 @@ def _no_draft_workflow(**kwargs): monkeypatch.setattr(AgentComposerService, "_workspace_dify_tools", lambda **kwargs: []) workflow_candidates = AgentComposerService.get_workflow_candidates( + variables=build_workflow_variable_service(database_client=sessionmaker(bind=session.get_bind())), session=session, tenant_id="tenant-1", app_id="app-1", node_id="node-1", user_id="account-1", + tool_providers=tool_providers, ) agent_app_candidates = AgentComposerService.get_agent_app_candidates( - session=session, tenant_id="tenant-1", agent_id="agent-1", user_id="account-1" + session=session, tenant_id="tenant-1", agent_id="agent-1", user_id="account-1", tool_providers=tool_providers ) impact = AgentComposerService.calculate_impact( session=session, tenant_id="tenant-1", current_snapshot_id="version-1" @@ -2673,8 +2685,8 @@ def fake_create_roster_agent_for_composer(**kwargs): lambda **kwargs: workflow_agent if kwargs["agent_id"] == workflow_agent.id else roster_agent, ) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: AgentConfigSnapshot( id="source-version-1", tenant_id="tenant-1", @@ -2849,10 +2861,10 @@ def test_node_job_only_updates_inline_agent_soul(monkeypatch: pytest.MonkeyPatch config_snapshot=AgentSoulConfig.model_validate({"prompt": {"system_prompt": "old"}}), ) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: current_snapshot) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: current_snapshot) monkeypatch.setattr(AgentComposerService, "_update_current_version", lambda **kwargs: next_snapshot) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: inline_agent) - monkeypatch.setattr(AgentComposerService, "_get_agent_draft", lambda **kwargs: normal_draft) + monkeypatch.setattr(AgentConfigRepository, "get_draft", lambda **kwargs: normal_draft) binding = WorkflowAgentNodeBinding( tenant_id="tenant-1", @@ -2953,7 +2965,7 @@ def test_get_or_create_normal_agent_draft_rebases_stale_workflow_only_draft(sqli session.add_all([agent, draft, active_snapshot]) session.commit() - resolved = AgentComposerService.get_or_create_normal_agent_draft( + resolved = AgentConfigRepository.normal_draft( session=session, tenant_id="tenant-1", agent=agent, @@ -2994,7 +3006,7 @@ def test_get_or_create_normal_agent_draft_keeps_roster_draft_edits(sqlite_sessio session.add_all([agent, draft]) session.commit() - resolved = AgentComposerService.get_or_create_normal_agent_draft( + resolved = AgentConfigRepository.normal_draft( session=session, tenant_id="tenant-1", agent=agent, @@ -3128,7 +3140,7 @@ def test_node_job_only_rejects_inline_binding_pointing_to_roster_agent( tenant_id="tenant-1", name="Agent", source=AgentSource.ROSTER, id="agent-1", scope=AgentScope.ROSTER ) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: current_snapshot) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: current_snapshot) monkeypatch.setattr(AgentComposerService, "_update_current_version", lambda **kwargs: next_snapshot) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) @@ -3226,7 +3238,7 @@ def test_copy_workflow_composer_from_roster_creates_inline_agent_and_preserves_n monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: workflow) monkeypatch.setattr(AgentComposerService, "_lock_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: source_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: source_version) def fake_create_workflow_only_agent(**kwargs): captured["create"] = kwargs @@ -3313,7 +3325,7 @@ def test_copy_workflow_composer_from_roster_rejects_stale_source_snapshot( ) session.commit() monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: roster_agent) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: source_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: source_version) with pytest.raises(AgentVersionConflictError): AgentComposerService.copy_workflow_composer_from_roster( @@ -3356,7 +3368,7 @@ def test_copy_workflow_composer_from_roster_rejects_unpublished_source( monkeypatch.setattr(AgentComposerService, "_lock_workflow_binding", lambda **kwargs: binding) monkeypatch.setattr(AgentComposerService, "_require_agent", lambda **kwargs: source_agent) require_version = MagicMock(side_effect=AssertionError("unpublished source must fail before loading snapshot")) - monkeypatch.setattr(AgentComposerService, "_require_version", require_version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", require_version) with pytest.raises(InvalidComposerConfigError, match="published config snapshot"): AgentComposerService.copy_workflow_composer_from_roster( @@ -3407,8 +3419,8 @@ def test_copy_workflow_composer_from_roster_is_idempotent_when_already_inline( session = sqlite_session monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: _workflow()) monkeypatch.setattr(AgentComposerService, "_lock_workflow_binding", lambda **kwargs: inline_binding) - monkeypatch.setattr(AgentComposerService, "_get_agent_if_present", lambda **kwargs: inline_agent) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: inline_version) + monkeypatch.setattr(AgentConfigRepository, "get_agent", lambda **kwargs: inline_agent) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: inline_version) def serialize_workflow_state(**kwargs): serialize_calls.append(kwargs) @@ -3547,8 +3559,8 @@ def get_app_backing_agent(self, *, tenant_id, app_id): monkeypatch.setattr(composer_service, "AgentRosterService", FakeAgentRosterService) monkeypatch.setattr(AgentComposerService, "_require_account", lambda **kwargs: _account()) monkeypatch.setattr( - AgentComposerService, - "_require_version", + AgentConfigRepository, + "require_snapshot", lambda **kwargs: AgentConfigSnapshot( version=1, config_snapshot=AgentSoulConfig(), @@ -3687,7 +3699,7 @@ def test_agent_app_draft_match_does_not_mark_create_version_as_published( tenant_id="tenant-1", agent_id="agent-1", version=1, config_snapshot=agent_soul, home_snapshot_id="home-1" ) session = sqlite_session - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: snapshot) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: snapshot) assert ( AgentComposerService._agent_soul_matches_active_config( @@ -3727,7 +3739,7 @@ def test_agent_app_draft_match_marks_publish_visible_revision_as_published( ) ) session.commit() - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **kwargs: snapshot) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **kwargs: snapshot) assert ( AgentComposerService._agent_soul_matches_active_config( @@ -3790,9 +3802,9 @@ def test_composer_version_helpers_and_lookup_errors(monkeypatch: pytest.MonkeyPa ) with pytest.raises(composer_service.AgentNotFoundError): AgentComposerService._require_agent(session=session, tenant_id="tenant-1", agent_id=None) - assert AgentComposerService._get_agent_if_present(session=session, tenant_id="tenant-1", agent_id="missing") is None + assert AgentConfigRepository.get_agent(session=session, tenant_id="tenant-1", agent_id="missing") is None assert ( - AgentComposerService._require_version( + AgentConfigRepository.require_snapshot( session=session, tenant_id="tenant-1", agent_id="agent-1", @@ -3801,7 +3813,7 @@ def test_composer_version_helpers_and_lookup_errors(monkeypatch: pytest.MonkeyPa == "version-1" ) with pytest.raises(composer_service.AgentVersionNotFoundError): - AgentComposerService._require_version( + AgentConfigRepository.require_snapshot( session=session, tenant_id="tenant-1", agent_id="agent-1", @@ -3844,7 +3856,7 @@ def test_composer_current_version_and_error_paths(monkeypatch: pytest.MonkeyPatc home_snapshot_id="home-1", config_snapshot='{"prompt":{"system_prompt":"old"}}', ) - monkeypatch.setattr(AgentComposerService, "_require_version", lambda **kwargs: version) + monkeypatch.setattr(AgentConfigRepository, "require_snapshot", lambda **kwargs: version) monkeypatch.setattr( AgentComposerService, "_require_agent", @@ -5938,9 +5950,8 @@ def _sync_roster_agent_task_refs( session.add(existing_binding) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -5970,9 +5981,13 @@ def test_publish_validation_rejects_agent_soul_publish_only_errors(self, sqlite_ session.commit() with pytest.raises(InvalidComposerConfigError, match="human_involvement_not_referenced"): - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_keeps_workspace_skill_refs_when_config_files_are_present(self, sqlite_session: Session): @@ -6039,9 +6054,13 @@ def test_publish_validation_keeps_workspace_skill_refs_when_config_files_are_pre session.add_all([binding, agent, snapshot, skill, skill_version, skill_binding, archive_file]) session.commit() - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_rejects_dangling_agent_soul_config_refs(self, sqlite_session: Session): @@ -6063,9 +6082,13 @@ def test_publish_validation_rejects_dangling_agent_soul_config_refs(self, sqlite session.commit() with pytest.raises(WorkflowAgentNodeValidationError, match="skill_ref_dangling"): - WorkflowAgentPublishService.validate_agent_nodes_for_publish( - session=session, - draft_workflow=self._agent_workflow(), + WorkflowAgentPublishService.validate_publication_state( + build_workflow_agent_service(session).publication_state( + draft_workflow=workflow_binding_scope(self._agent_workflow()), + ), + skills=WorkflowAgentSkillReader( + SkillManagementService(session_maker=sessionmaker(bind=session.get_bind())) + ), ) def test_publish_validation_rejects_missing_config_assets(self): @@ -6152,9 +6175,8 @@ def test_projects_binding_declared_outputs_to_draft_graph_response(self, sqlite_ session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) node_data = graph["nodes"][0]["data"] @@ -6213,9 +6235,8 @@ def test_projects_inline_binding_over_pending_inline_graph_response(self, sqlite session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) assert graph["nodes"][0]["data"]["agent_binding"] == { @@ -6268,9 +6289,8 @@ def test_keeps_pending_inline_graph_response_over_existing_roster_binding(self, session.add(binding) session.commit() - graph = WorkflowAgentPublishService.project_draft_bindings_to_graph( - session=session, - draft_workflow=workflow, + graph = build_workflow_agent_service(session).project_draft_bindings_to_graph( + draft_workflow=workflow_binding_scope(workflow), ) assert graph["nodes"][0]["data"]["agent_binding"] == { @@ -6324,9 +6344,8 @@ def test_creates_roster_binding_from_agent_node_graph(self, sqlite_session: Sess session.add_all([agent, self._publish_revision()]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6449,9 +6468,8 @@ def test_creates_inline_binding_from_agent_node_graph(self, sqlite_session: Sess session.add_all([agent, snapshot]) session.commit() - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6506,9 +6524,8 @@ def test_keeps_pending_inline_binding_in_draft_graph_without_db_binding(self, sq session.add(existing_binding) session.commit() - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6574,9 +6591,8 @@ def test_clones_inline_binding_for_agent_owned_by_another_node( ), ) - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6616,9 +6632,8 @@ def test_rejects_agent_node_graph_binding_with_unsupported_type(self, sqlite_ses ) with pytest.raises(ValueError, match="unsupported agent_binding type"): - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6649,9 +6664,8 @@ def test_treats_partial_inline_binding_as_pending_draft_state(self, sqlite_sessi ), ) - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6701,9 +6715,8 @@ def test_rejects_inline_binding_with_missing_snapshot(self, sqlite_session: Sess session.commit() with pytest.raises(ValueError, match="missing inline agent config snapshot"): - WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6764,9 +6777,8 @@ def test_updates_existing_roster_binding_prompt_from_agent_node_graph(self, sqli session.add_all([agent, self._publish_revision(), existing_binding]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -6837,9 +6849,8 @@ def test_updates_existing_roster_binding_declared_outputs_from_agent_node_graph( session.add_all([agent, self._publish_revision(), existing_binding]) session.commit() - WorkflowAgentPublishService.sync_roster_agent_bindings_for_draft( - session=session, - draft_workflow=workflow, + build_workflow_agent_service(session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -7038,9 +7049,8 @@ def test_deletes_draft_binding_and_returns_only_replaced_inline_agent( ] ) sqlite_session.commit() - retirement_candidates = WorkflowAgentPublishService.sync_agent_bindings_for_draft( - session=sqlite_session, - draft_workflow=workflow, + retirement_candidates = build_workflow_agent_service(sqlite_session)._synchronize_bindings( + draft_workflow=workflow_binding_scope(workflow), account_id="account-1", ) @@ -7221,7 +7231,7 @@ def count_flush(_session: Session, _flush_context: object) -> None: ) ), ) - monkeypatch.setattr(AgentComposerService, "_get_version_if_present", lambda **_kwargs: active_version) + monkeypatch.setattr(AgentConfigRepository, "get_snapshot", lambda **_kwargs: active_version) monkeypatch.setattr(AgentComposerService, "load_agent_composer", lambda **_kwargs: {"loaded": True}) payload = ComposerSavePayload.model_validate( @@ -7263,7 +7273,9 @@ def count_flush(_session: Session, _flush_context: object) -> None: assert not session.dirty -def test_workspace_dify_tools_returns_provider_and_tool_granularities(monkeypatch: pytest.MonkeyPatch): +def test_workspace_dify_tools_returns_provider_and_tool_granularities( + monkeypatch: pytest.MonkeyPatch, *, tool_providers +): """The slash-menu Tools tab needs both selection granularities: a provider hosts many tools (like an MCP server), so candidates return one provider-level entry (id = /*, = all tools) plus one per tool.""" @@ -7280,15 +7292,17 @@ def test_workspace_dify_tools_returns_provider_and_tool_granularities(monkeypatc ], ) - import services.tools.legacy_builtin_tools_manage_service as builtin_tools_module + import services.tools.builtin_tools_manage_service as builtin_tools_module monkeypatch.setattr( builtin_tools_module.BuiltinToolManageService, "list_builtin_tools", - staticmethod(lambda user_id, tenant_id: [provider]), + staticmethod(lambda user_id, tenant_id, **_kwargs: [provider]), ) - entries = AgentComposerService._workspace_dify_tools(tenant_id="tenant-1", user_id="user-1") + entries = AgentComposerService._workspace_dify_tools( + tenant_id="tenant-1", user_id="user-1", tool_providers=tool_providers + ) assert entries[0] == { "id": "duckduckgo/*", diff --git a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py index 2e56a28e2e48b1..e5585e2359a3d5 100644 --- a/api/tests/unit_tests/services/agent/test_tool_invocation_service.py +++ b/api/tests/unit_tests/services/agent/test_tool_invocation_service.py @@ -5,17 +5,27 @@ from unittest.mock import create_autospec, patch import pytest +from sqlalchemy import Connection, Engine, event from sqlalchemy.orm import Session, sessionmaker from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom +from core.tools.__base.tool import Tool from core.tools.__base.tool_runtime import ToolRuntime -from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage, ToolProviderType +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ( + ToolEntity, + ToolIdentity, + ToolInvokeFrom, + ToolInvokeMessage, + ToolProviderType, +) from core.tools.errors import ( ToolInvokeError, ToolParameterValidationError, ToolProviderCredentialValidationError, ToolProviderNotFoundError, ) +from core.tools.plugin_tool.tool import PluginTool from extensions.application_services.workflow import WorkflowExecutionDependencies from models.enums import AppStatus from models.model import App, AppMode @@ -25,6 +35,7 @@ from services.entities.agent_tool_inner import AgentToolInvokeRequest from services.errors.agent_tool_inner import AgentToolInnerServiceError from services.tools.agent_invocation_gateway import AgentToolInvocationGateway +from services.tools.tool_manager import ToolManager from services.workflow.execution.ports import WorkflowRuntime, WorkflowToolInvoker from services.workflow.variable_contracts import WorkflowExecutionVariables @@ -118,6 +129,72 @@ def _service( ) +@pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) +@pytest.mark.parametrize("fail_during_stream", [False, True]) +def test_app_lookup_releases_connection_before_tool_setup_and_stream_consumption( + sqlite_session: Session, + sqlite_engine: Engine, + workflow_runtime: WorkflowRuntime, + monkeypatch: pytest.MonkeyPatch, + fail_during_stream: bool, +) -> None: + _persist_app(sqlite_session) + active: set[Connection] = set() + begin, finish = active.add, active.discard + event.listen(sqlite_engine, "begin", begin) + event.listen(sqlite_engine, "commit", finish) + event.listen(sqlite_engine, "rollback", finish) + tool = PluginTool( + entity=ToolEntity( + identity=ToolIdentity( + author="author", name="search", provider="provider", label=I18nObject(en_US="Search") + ), + parameters=[], + ), + runtime=ToolRuntime(tenant_id=TENANT_ID, invoke_from=InvokeFrom.SERVICE_API, runtime_parameters={}), + tenant_id=TENANT_ID, + icon="icon.svg", + plugin_unique_identifier="plugin-id", + ) + calls = [] + + def resolve_runtime(**_kwargs: object) -> Tool: + assert not active + calls.append("setup") + return tool + + class PluginTransport: + def __init__(self, *_args: object, **_kwargs: object) -> None: + pass + + def invoke(self, **kwargs: object) -> Generator[ToolInvokeMessage]: + assert not active + assert kwargs["tenant_id"] == TENANT_ID + assert kwargs["app_id"] == APP_ID + calls.append("stream") + yield tool.create_text_message("ok") + assert not active + if fail_during_stream: + raise ToolInvokeError("stream interrupted") + + monkeypatch.setattr(ToolManager, "get_agent_tool_runtime", resolve_runtime) + monkeypatch.setattr("core.tools.plugin_tool.tool.PluginToolManager", PluginTransport) + try: + service = _service(sqlite_session, workflow_runtime) + if fail_during_stream: + with pytest.raises(AgentToolInnerServiceError) as raised: + service.invoke(_request()) + assert raised.value.error_code == "agent_tool_invoke_failed" + else: + assert service.invoke(_request()).observation == "ok" + assert calls == ["setup", "stream"] + assert not active + finally: + event.remove(sqlite_engine, "begin", begin) + event.remove(sqlite_engine, "commit", finish) + event.remove(sqlite_engine, "rollback", finish) + + @pytest.mark.parametrize("sqlite_session", [(App,)], indirect=True) @pytest.mark.parametrize( ("workflow_id", "expected_tool_invoke_from"), diff --git a/api/tests/unit_tests/services/annotation/test_reply_service.py b/api/tests/unit_tests/services/annotation/test_reply_service.py index 40ebad8a40dc0d..5cc598e2e0e02c 100644 --- a/api/tests/unit_tests/services/annotation/test_reply_service.py +++ b/api/tests/unit_tests/services/annotation/test_reply_service.py @@ -148,6 +148,30 @@ def search_by_vector( return AnnotationReplyService(AnnotationReplyRepository(store.sessions), AnnotationVectorRetrieval()) +def test_query_returns_none_when_annotation_setting_is_missing( + annotation_store: AnnotationStore, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with annotation_store.sessions.begin() as session: + setting = session.get(AppAnnotationSetting, "setting-1") + assert setting is not None + session.delete(setting) + + assert query(install_vector(monkeypatch, annotation_store)) is None + + +def test_query_returns_none_when_collection_binding_is_missing( + annotation_store: AnnotationStore, + monkeypatch: pytest.MonkeyPatch, +) -> None: + with annotation_store.sessions.begin() as session: + binding = session.get(DatasetCollectionBinding, "binding-1") + assert binding is not None + session.delete(binding) + + assert query(install_vector(monkeypatch, annotation_store)) is None + + @pytest.mark.parametrize("source", list(ConversationFromSource)) def test_search_releases_connection_and_commits_hit( annotation_store: AnnotationStore, diff --git a/api/tests/unit_tests/services/app/test_agent_app_feature_gateway.py b/api/tests/unit_tests/services/app/test_agent_app_feature_gateway.py index b181ad2c0c9c21..407ebbf978fae9 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_feature_gateway.py +++ b/api/tests/unit_tests/services/app/test_agent_app_feature_gateway.py @@ -7,6 +7,8 @@ touching model / prompt / agent_mode. """ +from unittest.mock import create_autospec + import pytest from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker @@ -19,6 +21,7 @@ from models.model import App, AppMode, AppModelConfig from services.app.agent_app_contracts import AgentAppNotFoundError from services.app.agent_app_feature_gateway import AgentAppFeatureValidator +from services.workflow.variable_contracts import WorkflowExecutionVariables TENANT_ID = "11111111-1111-1111-1111-111111111111" APP_ID = "22222222-2222-2222-2222-222222222222" @@ -110,7 +113,10 @@ def agent_app(sqlite_session: Session) -> App: def test_feature_save_is_atomic( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + ).features context = RequestContext("request", None, ACCOUNT_ID, TENANT_ID) calls: list[str] = [] @@ -144,7 +150,10 @@ def fail(_sender: App, **_kwargs: object) -> None: def test_feature_validation_failure_does_not_write( agent_app: App, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + ).features with pytest.raises(ValueError, match="opening_statement"): service.update_features( RequestContext("request", None, ACCOUNT_ID, TENANT_ID), "agent", {"opening_statement": 123} @@ -171,7 +180,10 @@ def test_features_require_owned_active_agent_app( else: agent.status = AgentStatus.ARCHIVED sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).features + service = build_agent_app_services( + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + ).features with pytest.raises(AgentAppNotFoundError): service.update_features(RequestContext("request", None, ACCOUNT_ID, TENANT_ID), "agent", {}) assert sqlite_session.scalar(select(AppModelConfig)) is None diff --git a/api/tests/unit_tests/services/app/test_agent_app_service.py b/api/tests/unit_tests/services/app/test_agent_app_service.py index dde98971924bdc..081b6096a87940 100644 --- a/api/tests/unit_tests/services/app/test_agent_app_service.py +++ b/api/tests/unit_tests/services/app/test_agent_app_service.py @@ -1,22 +1,18 @@ """Published Agent App references use the canonical roster query.""" +from unittest.mock import create_autospec + import pytest from sqlalchemy.orm import Session, sessionmaker +from enums.agent import WorkflowAgentBindingType from extensions.application_services.agent import build_agent_app_services from machinery.context import RequestContext -from models.agent import ( - Agent, - AgentKind, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentKind, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.agent_config_entities import WorkflowNodeJobConfig from models.model import AppMode from services.app.agent_app_contracts import AgentAppNotFoundError +from services.workflow.variable_contracts import WorkflowExecutionVariables from tests.helpers.unit.model_factories import make_app @@ -62,7 +58,10 @@ def test_access_only_lists_current_published_workflow_references( ) ) sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).access + service = build_agent_app_services( + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + ).access context = RequestContext("request", None, "account", "tenant-1") [reference] = service.list_referencing_workflows(context, "agent") assert reference["app_id"] == "workflow-app" @@ -97,6 +96,9 @@ def test_access_requires_public_agent_app( agent.app_id = "missing" sqlite_session.add_all([make_app(app_id="agent-app", mode=AppMode.AGENT), agent]) sqlite_session.commit() - service = build_agent_app_services(database_client=sqlite_session_factory).access + service = build_agent_app_services( + database_client=sqlite_session_factory, + variables=create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True), + ).access with pytest.raises(AgentAppNotFoundError): service.list_referencing_workflows(RequestContext("request", None, "account", "tenant-1"), "agent") diff --git a/api/tests/unit_tests/services/app/test_agent_chat_attachments.py b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py new file mode 100644 index 00000000000000..aa9c1025a26489 --- /dev/null +++ b/api/tests/unit_tests/services/app/test_agent_chat_attachments.py @@ -0,0 +1,263 @@ +"""Agent Chat reads image metadata through its injected database before accessing storage.""" + +import contextvars +from collections.abc import Iterator +from dataclasses import dataclass +from typing import cast +from unittest.mock import create_autospec +from uuid import uuid4 + +import pytest +from flask import Flask +from sqlalchemy import Engine, Table, create_engine +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import QueuePool + +from core.agent.entities import AgentEntity +from core.app.app_config.entities import DatasetRetrieveConfigEntity +from core.app.apps.base_app_queue_manager import AppQueueManager +from core.app.entities.app_invoke_entities import ( + AgentChatAppGenerateEntity, + InvokeFrom, + ModelConfigWithCredentialsEntity, +) +from core.db import session_factory +from core.model_manager import ModelInstance +from core.tools.__base.tool import Tool +from core.tools.__base.tool_runtime import ToolRuntime +from core.tools.entities.tool_entities import ToolInvokeMeta +from extensions.ext_database import db +from graphon.file import File, file_manager +from graphon.model_runtime.entities.message_entities import ImagePromptMessageContent +from graphon.model_runtime.entities.model_entities import AIModelEntity, ModelFeature +from models.annotation_reply import AnnotationReplies +from models.base import TypeBase +from models.model import App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile +from repositories.app.generation_repository import AppGenerationRepository +from services.agent.chat.function_call_runner import FunctionCallAgentRunner +from services.agent.chat.ports import AgentDatasetTools, AgentToolInvoker +from services.app.generation.adapters import agent_chat as worker_module +from services.app.generation.adapters.agent_chat import AgentChatAppGenerator +from services.app.generation.ports import AgentMessageRecords +from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler +from services.tools.dataset.tool import DatasetRetrieverTool, DatasetToolRetrieval +from services.tools.tool_engine import ToolEngine +from tests.helpers.unit.model_factories import make_app, make_upload_file +from tests.unit_tests.core.agent.test_base_agent_runner import ( + _app_config, + _app_generate, + _conversation, + _message, + _tool_entity, +) + + +@dataclass +class LLM: + def get_model_schema(self, _name: str, _credentials: dict[str, object] | None) -> AIModelEntity: + return AIModelEntity.model_construct(features=[ModelFeature.VISION]) + + +@dataclass +class Model: + model_type_instance: LLM + model_name: str = "model" + credentials: dict[str, object] | None = None + + +@pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) +def agent_database( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Iterator[tuple[sessionmaker[Session], AppGenerationRepository, Engine]]: + engine = create_engine("sqlite://", poolclass=QueuePool) + TypeBase.metadata.create_all( + engine, + tables=[ + cast(Table, model.__table__) + for model in (App, Conversation, Message, MessageAgentThought, MessageFile, UploadFile) + ], + ) + assert isinstance(request.param, bool) + sessions = sessionmaker(engine, expire_on_commit=request.param) + with sessions.begin() as session: + session.add_all([make_app(app_id="app1", tenant_id="tenant"), _conversation(), _message()]) + + def forbidden(*_args: object, **_kwargs: object) -> None: + pytest.fail("Agent Chat accessed a global database") + + monkeypatch.setattr(session_factory, "create_session", forbidden) + monkeypatch.setattr(session_factory, "get_session_maker", forbidden) + monkeypatch.setattr(db, "session", forbidden) + monkeypatch.setattr(type(db), "engine", property(forbidden)) + try: + yield sessions, AppGenerationRepository(sessions), engine + finally: + engine.dispose() + + +def test_dataset_images_are_owned_ordered_and_materialized_before_storage( + agent_database: tuple[sessionmaker[Session], AppGenerationRepository, Engine], + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions, records, engine = agent_database + ids = [str(uuid4()) for _ in range(5)] + with sessions.begin() as session: + session.add_all( + [ + make_upload_file( + file_id=ids[0], + tenant_id="tenant", + name="first.png", + extension="png", + mime_type="image/png", + key="first", + ), + make_upload_file( + file_id=ids[1], + tenant_id="tenant", + name="second.png", + extension="png", + mime_type="image/png", + key="second", + ), + make_upload_file( + file_id=ids[2], tenant_id="foreign", extension="png", mime_type="image/png", key="foreign" + ), + make_upload_file(file_id=ids[3], tenant_id="tenant", mime_type="text/plain"), + ] + ) + config = _app_config() + runner = FunctionCallAgentRunner( + records=records, + dataset_tools=lambda **_kwargs: [], + tool_invoker=create_autospec(AgentToolInvoker, instance=True, spec_set=True), + tenant_id="tenant", + application_generate_entity=_app_generate(app_config=config), + conversation=_conversation(), + app_config=config, + model_config=ModelConfigWithCredentialsEntity.model_construct(), + config=AgentEntity(provider="provider", model="model", strategy=AgentEntity.Strategy.FUNCTION_CALLING), + queue_manager=create_autospec(AppQueueManager, instance=True, spec_set=True), + message=_message(), + user_id="user", + model_instance=cast(ModelInstance, Model(LLM())), + ) + reads = [] + + class FileRuntime: + multimodal_send_format = "base64" + + def load_file_bytes(self, *, file: File) -> bytes: + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + reads.append(file.filename) + return b"image bytes" + + monkeypatch.setattr(file_manager, "get_workflow_file_runtime", FileRuntime) + + class Retrieval: + def retrieve_dataset( + self, + *, + tenant_id: str, + app_id: str, + user_id: str, + dataset_id: str, + query: str, + config: DatasetRetrieveConfigEntity, + top_k: int, + inputs: dict[str, object], + invoke_from: InvokeFrom, + return_resource: bool, + hit_callback: DatasetIndexToolCallbackHandler, + ) -> str: + del tenant_id, app_id, user_id, dataset_id, config, top_k + del inputs, invoke_from, return_resource, hit_callback + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + return query + + tool = DatasetRetrieverTool( + entity=_tool_entity("dataset"), + runtime=ToolRuntime(tenant_id="tenant"), + retrieval=cast(DatasetToolRetrieval, Retrieval()), + dataset_id="dataset", + config=DatasetRetrieveConfigEntity(retrieve_strategy="single"), + top_k=2, + inputs={}, + invoke_from=InvokeFrom.DEBUGGER, + return_resource=False, + hit_callback=create_autospec(DatasetIndexToolCallbackHandler, instance=True, spec_set=True), + app_id="app1", + ) + response = "\n".join(f"![image](/files/{id}/file-preview)" for id in [ids[1], ids[0], *ids]) + with sessions() as tool_session: + text, files, meta = ToolEngine.agent_invoke( + session=tool_session, + records=records, + tool=tool, + tool_parameters={"query": response}, + user_id="user", + tenant_id="tenant", + message=_message(), + invoke_from=InvokeFrom.DEBUGGER, + agent_tool_callback=runner.agent_callback, + ) + assert files == [] + assert isinstance(meta, ToolInvokeMeta) + assert meta.error is None + contents = runner._build_dataset_tool_image_contents(text, tool) + assert reads == ["second.png", "first.png"] + assert all( + isinstance(content, ImagePromptMessageContent) and content.base64_data == "aW1hZ2UgYnl0ZXM=" + for content in contents + ) + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + + +def test_worker_passes_records_and_dataset_factory_without_opening_session( + agent_database: tuple[sessionmaker[Session], AppGenerationRepository, Engine], + monkeypatch: pytest.MonkeyPatch, + app: Flask, + annotation_replies: AnnotationReplies, +) -> None: + _, records, engine = agent_database + calls: list[str] = [] + expected_records = records + expected_tool_invoker = create_autospec(AgentToolInvoker, instance=True, spec_set=True) + + def datasets(**_kwargs: object) -> list[Tool]: + return [] + + class Runner: + def __init__( + self, *, records: AgentMessageRecords, dataset_tools: AgentDatasetTools, tool_invoker: AgentToolInvoker + ) -> None: + assert records is expected_records + assert dataset_tools is datasets + assert tool_invoker is expected_tool_invoker + + def run(self, application_generate_entity: AgentChatAppGenerateEntity, **kwargs: object) -> None: + assert "session" not in kwargs + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + calls.append(application_generate_entity.task_id) + + monkeypatch.setattr(worker_module, "AgentChatAppRunner", Runner) + generator = AgentChatAppGenerator( + dataset_tools=datasets, + tool_invoker=expected_tool_invoker, + records=records, + annotations=annotation_replies, + ) + entity = AgentChatAppGenerateEntity.model_construct(task_id="task", app_config=_app_config()) + generator._generate_worker( + app, + contextvars.copy_context(), + entity, + create_autospec(AppQueueManager, instance=True, spec_set=True), + "conv1", + "msg_current", + ) + assert calls == ["task"] diff --git a/api/tests/unit_tests/services/app/test_console_gateway.py b/api/tests/unit_tests/services/app/test_console_gateway.py index 947d2488d3935d..a54a3e3ae000a7 100644 --- a/api/tests/unit_tests/services/app/test_console_gateway.py +++ b/api/tests/unit_tests/services/app/test_console_gateway.py @@ -9,7 +9,6 @@ import pytest import yaml -from blinker import Namespace from sqlalchemy import Engine, event, select from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session, sessionmaker @@ -18,6 +17,7 @@ from core.plugin.impl.plugin import PluginInstaller from enums import CloudPlan, DeploymentEdition from extensions.application_services.app import AppServices +from extensions.application_services.workflow import build_app_dsl_service from machinery.context import AppRequestContext, RequestContext from models.account import Account, TenantAccountJoin, TenantAccountRole from models.agent import ( @@ -33,6 +33,7 @@ ) from models.agent_config_entities import AgentSoulConfig from models.model import App, AppMode, AppModelConfig, IconType +from models.tool_runtime_contracts import WorkflowToolQueries from models.workflow import Workflow, WorkflowType from repositories.app.export_repository import AppExportRepository from services.agent.dsl_entities import AGENT_PACKAGE_REF_KEY @@ -62,7 +63,7 @@ from services.plugin.dependencies_analysis import DependenciesAnalysisService from services.recommended_app_package_service import RecommendedAppPackageService from services.system_feature_service import SystemFeatureService -from services.workflow_service import WorkflowService +from services.tools.provider_queries import ToolProviders from tests.helpers.unit.audio_runtime_fixtures import record_calls from tests.helpers.unit.model_factories import make_account, make_tenant, make_upload_file from tests.helpers.unit.rbac_fakes import RBACDomain @@ -188,10 +189,6 @@ def test_copy_preserves_shared_roster_bindings_and_clones_owned_inline_agents( ) monkeypatch.setattr(DependenciesAnalysisService, "generate_dependencies", lambda **_kwargs: []) monkeypatch.setattr(SystemFeatureService, "is_webapp_auth_enabled", lambda: False) - workflow_service = WorkflowService(session_maker=sqlite_session_factory) - monkeypatch.setattr( - "services.app_dsl_service.WorkflowService", Mock(wraps=WorkflowService, return_value=workflow_service) - ) exported = app_services.console.export(context, app_id, AppExportOptions(format="yaml")) assert isinstance(exported, str) @@ -511,6 +508,8 @@ def serialize_export_data(prepared: AppDslExportData) -> str: agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()), + workflow_tools=Mock(), + workflow_publisher=Mock(), ) context = RequestContext("request", None, "actor", workspace) options = AppExportOptions(include_secret=True, workflow_id="workflow") @@ -585,11 +584,13 @@ def fetch_plugins(_self: PluginInstaller, tenant_id: str, plugin_ids: list[str]) monkeypatch.setattr(PluginInstaller, "fetch_plugin_installation_by_ids", fetch_plugins) gateway = AppTransferGateway( session_factory=factory, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()), + workflow_tools=Mock(), + workflow_publisher=Mock(), ) exports = AppExportService( apps=AppExportRepository(session_factory=factory), serialize=AppDslService.serialize_export_data @@ -803,6 +804,8 @@ def test_create_draft_saves_the_starter_graph_of_the_apps_mode( sqlite_session_factory: sessionmaker[Session], copy_source: tuple[RequestContext, str], monkeypatch: pytest.MonkeyPatch, + tool_providers: ToolProviders, + workflow_queries: WorkflowToolQueries, mode: AppMode, ) -> None: context, app_id = copy_source @@ -812,12 +815,15 @@ def test_create_draft_saves_the_starter_graph_of_the_apps_mode( app.mode = mode syncs: list[dict[str, object]] = [] monkeypatch.setattr( - WorkflowService, "sync_draft_workflow", record_calls(WorkflowService.sync_draft_workflow, syncs) + console_gateway.WorkflowDefinitionStore, + "sync_draft_workflow", + record_calls(console_gateway.WorkflowDefinitionStore.sync_draft_workflow, syncs), ) - # The app event handlers reach the Flask-bound database; this test only checks the saved draft. - monkeypatch.setattr("services.workflow_service.app_draft_workflow_was_synced", Namespace().signal("synced")) - - AppLifecycleGateway(session_factory=sqlite_session_factory).create_draft(context, app_id) + AppLifecycleGateway( + session_factory=sqlite_session_factory, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ).create_draft(context, app_id) assert [sync["graph"] for sync in syncs] == [console_gateway._STARTER_DRAFT_GRAPHS[mode]] assert syncs[0]["graph"] is not console_gateway._STARTER_DRAFT_GRAPHS[mode] @@ -906,6 +912,8 @@ def sync(tenant_id: str, account_id: str, resource_type: str, resource_id: str) agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), recommended_packages=RecommendedAppPackageService(sources=Mock(), exporter=Mock()), + workflow_tools=Mock(), + workflow_publisher=Mock(), ) import_entry = gateway.import_copy if entry == "copy" else gateway.import_dsl if commit_fails and status != ImportStatus.FAILED and not (entry == "copy" and status == ImportStatus.PENDING): diff --git a/api/tests/unit_tests/services/app/test_console_service.py b/api/tests/unit_tests/services/app/test_console_service.py index 2b89ba73883f8b..7cc8f8603f5b9e 100644 --- a/api/tests/unit_tests/services/app/test_console_service.py +++ b/api/tests/unit_tests/services/app/test_console_service.py @@ -9,6 +9,7 @@ from datetime import datetime from io import BytesIO from typing import BinaryIO, Literal, cast, override +from unittest.mock import Mock from uuid import UUID, uuid4 from zipfile import ZipFile @@ -22,10 +23,9 @@ from core.plugin.impl.plugin import PluginInstaller from core.rbac import RBACPermission from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolProviderType -from core.tools.tool_manager import ToolManager from events.app_event import app_was_updated from extensions.application_services.app import AppServices -from extensions.ext_database import db +from extensions.application_services.workflow import build_app_dsl_service from extensions.ext_redis import redis_client from graphon.model_runtime.entities.model_entities import ModelType from libs.datetime_utils import naive_utc_now @@ -40,6 +40,8 @@ from models.workflow import Workflow, WorkflowType from repositories.app.console_repository import ConsoleAppRepository from repositories.recommended_app_catalog_repository import DatabaseRecommendedAppCatalogRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.agent.errors import ( AgentAccessNotReadyError, InvalidRosterAgentPackageError, @@ -58,7 +60,6 @@ CreatorsPlatformDisabledError, InvalidAppExportError, ) -from services.app_dsl_service import AppDslService from services.app_package_service import AppPackageService from services.app_tracing_config_service import AppTraceSettings from services.entities.app_entities import ( @@ -90,6 +91,7 @@ from services.model_provider.service import ModelProviderService from services.plugin.dependencies_analysis import DependenciesAnalysisService from services.recommended_app_package_service import RecommendedAppPackageService +from services.tools.tool_manager import ToolManager from services.workflow_tool_references import WorkflowToolTarget from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task from tests.helpers.unit.model_factories import make_account, make_tenant @@ -563,6 +565,9 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( monkeypatch: pytest.MonkeyPatch, provider_name: str | None, expected_model: tuple[str, str, str] | None, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -598,7 +603,9 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( ) context = RequestContext("request", None, account.id, tenant.id) - app = AppLifecycleGateway(session_factory=factory).create( + app = AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ).create( context, CreateAppParams(name="Agent", mode=AppMode.AGENT.value), AppCreationSettings({"enable_site": False, "enable_api": False}, None), @@ -622,7 +629,12 @@ def test_lifecycle_agent_creation_seeds_workspace_default_model( @pytest.mark.parametrize("failure", [None, "persist", "external"]) @pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.AGENT]) def test_lifecycle_runs_after_atomic_creation_and_session_close( - sqlite_engine: Engine, failure: str | None, mode: AppMode + sqlite_engine: Engine, + failure: str | None, + mode: AppMode, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: @@ -710,7 +722,9 @@ def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=LifecycleProbe(session_factory=factory), + lifecycle=LifecycleProbe( + tool_providers=tool_providers, session_factory=factory, workflow_queries=workflow_queries + ), ) event.listen(sqlite_engine, "checkout", checkout) event.listen(sqlite_engine, "checkin", checkin) @@ -743,7 +757,12 @@ def deleted(self, context: RequestContext, deleted: AppDeletion) -> None: event.remove(sqlite_engine, "checkin", checkin) -def test_app_deletion_failure_rolls_back_backing_agent_changes(sqlite_engine: Engine) -> None: +def test_app_deletion_failure_rolls_back_backing_agent_changes( + sqlite_engine: Engine, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) with factory.begin() as session: account = Account(name="Creator", email=f"{uuid4()}@example.com") @@ -758,7 +777,9 @@ def test_app_deletion_failure_rolls_back_backing_agent_changes(sqlite_engine: En ] ) context = RequestContext("request", None, account.id, tenant.id) - gateway = AppLifecycleGateway(session_factory=factory) + gateway = AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ) app = gateway.create( context, CreateAppParams(name="Agent", mode="agent"), @@ -786,7 +807,13 @@ def reject_commit(_session: Session) -> None: @pytest.mark.parametrize("operation", ["get", "update", "create", "copy"]) @pytest.mark.parametrize("remote_fails", [False, True]) def test_detail_tool_enrichment_releases_database_before_plugin_io( - sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, operation: str, remote_fails: bool + sqlite_engine: Engine, + monkeypatch: pytest.MonkeyPatch, + operation: str, + remote_fails: bool, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) missing_api = str(uuid4()) @@ -894,7 +921,9 @@ def import_copy(self, context: RequestContext, params: AppImportParams) -> Impor transfers=CopyTransfer(), creators=Creators(), tracing=Tracing(), - lifecycle=LifecycleGateway(session_factory=factory), + lifecycle=LifecycleGateway( + tool_providers=tool_providers, session_factory=factory, workflow_queries=workflow_queries + ), ) monkeypatch.setattr(ToolManager, "get_hardcoded_provider", hardcoded) monkeypatch.setattr(PluginInstaller, "check_tools_existence", check_plugins) @@ -941,7 +970,14 @@ def read_result() -> AppRecord: @pytest.mark.parametrize("operation", ["rename", "icon", "site"]) @pytest.mark.parametrize("commit_fails", [False, True]) -def test_app_mutations_publish_only_after_commit(sqlite_engine: Engine, operation: str, commit_fails: bool) -> None: +def test_app_mutations_publish_only_after_commit( + sqlite_engine: Engine, + operation: str, + commit_fails: bool, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False, close_resets_only=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) with factory.begin() as session: @@ -962,7 +998,9 @@ def test_app_mutations_publish_only_after_commit(sqlite_engine: Engine, operatio transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=AppLifecycleGateway(session_factory=factory), + lifecycle=AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ), ) signals: list[str] = [] checked_out: set[object] = set() @@ -1027,7 +1065,13 @@ def update() -> AppRecord: @pytest.mark.parametrize("surface", ["site", "api"]) -def test_unpublished_agent_cannot_enable_access_through_console_service(sqlite_engine: Engine, surface: str) -> None: +def test_unpublished_agent_cannot_enable_access_through_console_service( + sqlite_engine: Engine, + surface: str, + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, +) -> None: factory = sessionmaker(bind=sqlite_engine, expire_on_commit=False) context = RequestContext("request", None, str(uuid4()), str(uuid4())) with factory.begin() as session: @@ -1053,7 +1097,9 @@ def test_unpublished_agent_cannot_enable_access_through_console_service(sqlite_e transfers=Transfers(), creators=Creators(), tracing=Tracing(), - lifecycle=AppLifecycleGateway(session_factory=factory), + lifecycle=AppLifecycleGateway( + session_factory=factory, tool_providers=tool_providers, workflow_queries=workflow_queries + ), ) signals: list[object] = [] @@ -1151,18 +1197,14 @@ def _tool_node(provider_id: str, tool_name: str) -> dict[str, object]: @pytest.fixture def workspaces( sqlite_session_factory: sessionmaker[Session], - sqlite_engine: Engine, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None], ) -> Workspaces: config_overrides(RBAC_ENABLED=False, DEPLOYMENT_EDITION="COMMUNITY") - # Workflow Tool management opens its own sessions from the Flask-SQLAlchemy engine. - monkeypatch.setattr(type(db), "engine", property(lambda _: sqlite_engine)) # Plugin metadata is resolved by the plugin daemon; these Apps use no plugins. monkeypatch.setattr(DependenciesAnalysisService, "generate_dependencies", _no_plugin_dependencies) # Workflow event handlers reach the Flask-bound database; these tests check the imported records. - monkeypatch.setattr("services.workflow_service.app_draft_workflow_was_synced", Signal()) - monkeypatch.setattr("services.workflow_service.app_published_workflow_was_updated", Signal()) + monkeypatch.setattr("services.workflow.definition_gateway.app_draft_workflow_was_synced", Signal()) actor, source_id, target_id = str(uuid4()), str(uuid4()), str(uuid4()) caller_app_id, tool_app_id, provider_id = str(uuid4()), str(uuid4()), str(uuid4()) version = Workflow.version_from_datetime(naive_utc_now()) @@ -1603,7 +1645,7 @@ def test_planning_new_workflow_tools_checks_tool_management_under_rbac( rbac_domain.transport.response = {"allowed": allowed} gateway = AppTransferGateway( session_factory=sqlite_session_factory, - dsl_factory=AppDslService, + dsl_factory=build_app_dsl_service, packages=AppPackageService(), agent_packages=RosterAgentPackageExporter(), agent_importer=RosterAgentPackageImporter(), @@ -1611,6 +1653,8 @@ def test_planning_new_workflow_tools_checks_tool_management_under_rbac( sources=DatabaseRecommendedAppCatalogRepository(session_factory=sqlite_session_factory, redis=redis_client), exporter=RosterAgentPackageExporter(), ), + workflow_tools=WorkflowToolRepository(sqlite_session_factory), + workflow_publisher=Mock(), ) tool = PackagedWorkflowTool( id="w_000001", diff --git a/api/tests/unit_tests/services/app/test_response_gateway.py b/api/tests/unit_tests/services/app/test_response_gateway.py index 95512bced132fa..52a72ad4d8321b 100644 --- a/api/tests/unit_tests/services/app/test_response_gateway.py +++ b/api/tests/unit_tests/services/app/test_response_gateway.py @@ -2,11 +2,15 @@ from machinery.context import RequestContext from models.model import AppMode +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.app.response_gateway import AppResponseGateway from services.entities.app_entities import AppRecord -def test_mask_record_omits_unverified_tool_parameters_without_mutating_record() -> None: +def test_mask_record_omits_unverified_tool_parameters_without_mutating_record( + tool_providers: ToolProviderRepository, workflow_queries: WorkflowToolRepository +) -> None: agent_mode = { "enabled": True, "tools": [ @@ -26,7 +30,9 @@ def test_mask_record_omits_unverified_tool_parameters_without_mutating_record() ) context = RequestContext("request", None, "account", "tenant") - masked = AppResponseGateway.mask_record(context, app) + masked = AppResponseGateway(tool_providers=tool_providers, workflow_queries=workflow_queries).mask_record( + context, app + ) assert masked is not app assert masked.app_model_config is not None diff --git a/api/tests/unit_tests/services/data_migration/test_export_service.py b/api/tests/unit_tests/services/data_migration/test_export_service.py index faad5d2052114c..a29bccc24ffcbf 100644 --- a/api/tests/unit_tests/services/data_migration/test_export_service.py +++ b/api/tests/unit_tests/services/data_migration/test_export_service.py @@ -98,10 +98,13 @@ def test_export_config_parser_rejects_unsupported_app_modes(): ) -def test_secret_free_api_tool_export_uses_masking_and_omits_credentials(monkeypatch: pytest.MonkeyPatch): +def test_secret_free_api_tool_export_uses_masking_and_omits_credentials( + monkeypatch: pytest.MonkeyPatch, *, workflow_tools, tool_providers +): calls = [] - def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): + def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True, *, tool_providers): + assert tool_providers is service._tool_providers calls.append((provider, tenant_id, mask)) return {"credentials": {"api_key": "masked"}, "schema": {"openapi": "3.0.0"}, "tools": ["unused"]} @@ -109,7 +112,7 @@ def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): "services.data_migration.export_service.ToolManager.user_get_api_provider", fake_get_api_provider, ) - service = MigrationExportService() + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) tools: list[dict] = [] report_items = [] @@ -127,8 +130,8 @@ def fake_get_api_provider(provider: str, tenant_id: str, mask: bool = True): @pytest.mark.parametrize("sqlite_session", [()], indirect=True) -def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Session): - service = MigrationExportService() +def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Session, *, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) dependencies: list[dict] = [] mcp_tools: list[dict] = [] report_items = [] @@ -158,7 +161,9 @@ def test_secret_free_mcp_dependencies_are_dependency_only(sqlite_session: Sessio @pytest.mark.parametrize("sqlite_session", [(MCPToolProvider,)], indirect=True) -def test_get_mcp_provider_does_not_compare_non_uuid_identifier_to_uuid_id(sqlite_session: Session): +def test_get_mcp_provider_does_not_compare_non_uuid_identifier_to_uuid_id( + sqlite_session: Session, *, workflow_tools, tool_providers +): sqlite_session.add( MCPToolProvider( name="Other tenant provider", @@ -184,7 +189,9 @@ def capture_statement(_conn, _cursor, statement, _parameters, _context, _execute try: with pytest.raises(MigrationDataError, match="MCP provider not found"): - MigrationExportService()._get_mcp_provider(_TENANT_ID, "my-test-mcp", session=sqlite_session) + MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers)._get_mcp_provider( + _TENANT_ID, "my-test-mcp", session=sqlite_session + ) finally: event.remove(bind, "before_cursor_execute", capture_statement) @@ -193,8 +200,8 @@ def capture_statement(_conn, _cursor, statement, _parameters, _context, _execute assert "tool_mcp_providers.server_identifier =" in statements[0] -def test_dependency_ids_are_deduplicated_with_manual_selection_first(): - service = MigrationExportService() +def test_dependency_ids_are_deduplicated_with_manual_selection_first(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) provider_ids = service._provider_ids( manual_provider_ids=["weather", "weather", "manual"], discovered_dependencies=[ @@ -208,8 +215,8 @@ def test_dependency_ids_are_deduplicated_with_manual_selection_first(): assert provider_ids == ["weather", "manual", "forecast"] -def test_api_provider_ids_use_provider_name_from_discovered_dependencies(): - service = MigrationExportService() +def test_api_provider_ids_use_provider_name_from_discovered_dependencies(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) provider_ids = service._provider_ids( manual_provider_ids=[], discovered_dependencies=[ @@ -221,8 +228,8 @@ def test_api_provider_ids_use_provider_name_from_discovered_dependencies(): assert provider_ids == ["weather"] -def test_mcp_authentication_export_omits_runtime_header_shape(): - service = MigrationExportService() +def test_mcp_authentication_export_omits_runtime_header_shape(*, workflow_tools, tool_providers): + service = MigrationExportService(workflow_tools=workflow_tools, tool_providers=tool_providers) assert service._serialize_mcp_authentication({"Authorization": "Bearer token"}) is None assert service._serialize_mcp_authentication({"client_id": "id", "client_secret": "secret"}) == { diff --git a/api/tests/unit_tests/services/data_migration/test_import_service.py b/api/tests/unit_tests/services/data_migration/test_import_service.py index 5cc2c96e1f670c..73e3055d6fa130 100644 --- a/api/tests/unit_tests/services/data_migration/test_import_service.py +++ b/api/tests/unit_tests/services/data_migration/test_import_service.py @@ -1,8 +1,6 @@ -from contextlib import nullcontext from dataclasses import dataclass -from types import SimpleNamespace from typing import cast -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec import pytest import yaml @@ -26,23 +24,31 @@ ResourceReportItem, ResourceType, ) -from services.data_migration.import_service import ImportRequest, ImportTargetResolver, MigrationImportService +from services.data_migration.import_service import ( + ImportRequest, + ImportTargetResolver, + MigrationImportService, + WorkflowPublisher, +) from services.entities.dsl_entities import ImportStatus from tests.helpers.unit.config_override import apply_config_overrides @dataclass(frozen=True) class Database: - """Typed database binding used by import code that still reads ``db.engine``.""" + """Explicit SQLite binding shared by migration tests.""" engine: Engine session: Session +def _workflow_publisher() -> WorkflowPublisher: + return create_autospec(WorkflowPublisher, instance=True, spec_set=True) + + @pytest.fixture -def database(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> Database: +def database(sqlite_session: Session) -> Database: database = Database(engine=cast(Engine, sqlite_session.get_bind()), session=sqlite_session) - monkeypatch.setattr(import_service, "db", database) return database @@ -236,7 +242,7 @@ def test_package_target_tenant_id_ignores_invalid_uuid(database: Database): assert database.session.query(Tenant).count() == 0 -def test_options_override_replaces_package_defaults(database: Database): +def test_options_override_replaces_package_defaults(database: Database, *, workflow_tools): package = MigrationPackage.from_mapping( { "metadata": { @@ -275,7 +281,12 @@ def _import_workflows( override = ImportOptions(create_app_api_token_on_import=False, conflict_strategy=ConflictStrategy.SKIP) - CapturingImportService(target_resolver=StubResolver()).import_package( + CapturingImportService( + app_dsl=Mock(), + workflows=_workflow_publisher(), + target_resolver=StubResolver(), + workflow_tools=workflow_tools, + ).import_package( ImportRequest(package=package, options_override=override), session=database.session, ) @@ -283,19 +294,24 @@ def _import_workflows( assert captured_options == [override] -def test_only_preserve_id_strategy_reuses_source_app_id(): - service = MigrationImportService() +def test_only_preserve_id_strategy_reuses_source_app_id(*, workflow_tools): + service = MigrationImportService(app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools) assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.PRESERVE_ID)) is True assert service._should_preserve_source_app_id(ImportOptions(id_strategy=IdStrategy.GENERATE_NEW_ID)) is False -def test_find_existing_app_ignores_invalid_uuid(database: Database): +def test_find_existing_app_ignores_invalid_uuid(database: Database, *, workflow_tools): _persist_app(database.session, app_id="app-other", tenant_id="tenant-2") - assert MigrationImportService()._find_existing_app("not-a-uuid", "tenant-1", session=database.session) is None + assert ( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._find_existing_app("not-a-uuid", "tenant-1", session=database.session) + is None + ) -def test_find_existing_workflow_tool_does_not_compare_invalid_uuid(database: Database): +def test_find_existing_workflow_tool_does_not_compare_invalid_uuid(database: Database, *, workflow_tools): provider = _persist_workflow_provider(database.session, provider_id="provider-1", app_id="app-id", name="tool-name") statements = [] @@ -304,9 +320,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_existing_workflow_tool( - "tenant-1", "not-a-uuid", "tool-name", "app-id", session=database.session - ) + result = MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._find_existing_workflow_tool("tenant-1", "not-a-uuid", "tool-name", "app-id", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -315,7 +331,7 @@ def capture_statement(orm_execute_state) -> None: assert f"{WorkflowToolProvider.__tablename__}.id" not in where_clause -def test_find_existing_mcp_tool_does_not_compare_invalid_uuid(database: Database): +def test_find_existing_mcp_tool_does_not_compare_invalid_uuid(database: Database, *, workflow_tools): provider = _persist_mcp_provider(database.session, provider_id="provider-1") statements = [] @@ -324,9 +340,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_existing_mcp_tool( - "tenant-1", "my-test-mcp", "my-test-mcp", session=database.session - ) + result = MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._find_existing_mcp_tool("tenant-1", "my-test-mcp", "my-test-mcp", session=database.session) finally: event.remove(database.session, "do_orm_execute", capture_statement) @@ -337,22 +353,24 @@ def capture_statement(orm_execute_state) -> None: def test_workflow_app_import_closes_read_transaction_before_dsl_overwrite( - monkeypatch: pytest.MonkeyPatch, database: Database + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools ): class StubAppDslService: def __init__(self, session): + assert session is database.session self.session = session def import_app(self, **kwargs): assert not self.session.in_transaction() return Import(id="import-id", status=ImportStatus.COMPLETED, app_id="imported-app-id") - monkeypatch.setattr(import_service, "AppDslService", StubAppDslService) apply_config_overrides(monkeypatch, RBAC_ENABLED=True) existing_app = _persist_app(database.session, app_id="11111111-1111-4111-8111-111111111111") database.session.begin() - imported_app_id = MigrationImportService()._import_workflow_app( + imported_app_id = MigrationImportService( + app_dsl=StubAppDslService, workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_workflow_app( account=object(), workflow_data={"name": "main_chatflow"}, dsl_content="app:\n mode: workflow\n", @@ -365,7 +383,7 @@ def import_app(self, **kwargs): assert imported_app_id == "imported-app-id" -def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): +def test_rewrite_workflow_dsl_replaces_tool_provider_ids(*, workflow_tools): dsl_content = yaml.safe_dump( { "app": {"mode": "workflow"}, @@ -400,7 +418,9 @@ def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): } ) - rewritten = MigrationImportService()._rewrite_workflow_dsl_provider_ids( + rewritten = MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._rewrite_workflow_dsl_provider_ids( dsl_content, { "source-api-provider-id": "target-api-provider-id", @@ -415,7 +435,7 @@ def test_rewrite_workflow_dsl_replaces_tool_provider_ids(): ) -def test_source_api_provider_ids_are_discovered_from_workflow_dsl(): +def test_source_api_provider_ids_are_discovered_from_workflow_dsl(*, workflow_tools): package = MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -444,11 +464,13 @@ def test_source_api_provider_ids_are_discovered_from_workflow_dsl(): } ) - assert MigrationImportService()._source_api_provider_ids_by_name(package) == {"weather": {"source-api-provider-id"}} + assert MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._source_api_provider_ids_by_name(package) == {"weather": {"source-api-provider-id"}} def test_workflow_tool_import_publishes_referenced_app_before_create( - monkeypatch: pytest.MonkeyPatch, database: Database + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools ): events = [] _persist_tenant_account(database.session) @@ -467,8 +489,10 @@ def create_workflow_tool(**kwargs) -> None: app_id=kwargs["workflow_app_id"], ) - monkeypatch.setattr(import_service.WorkflowToolManageService, "create_workflow_tool", create_workflow_tool) - PublishingImportService()._import_workflow_tools( + monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) + PublishingImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -486,33 +510,29 @@ def create_workflow_tool(**kwargs) -> None: assert events == [("published", app_id), ("created", app_id)] -def test_ensure_workflow_app_is_published_updates_current_workflow( - monkeypatch: pytest.MonkeyPatch, - database: Database, -) -> None: +def test_ensure_workflow_app_is_published_updates_current_workflow(database: Database, *, workflow_tools) -> None: _, account = _persist_tenant_account(database.session) app_id = "00000000-0000-0000-0000-000000000001" _persist_app(database.session, app_id=app_id) - publish = Mock(return_value=SimpleNamespace(id="published-workflow")) - monkeypatch.setattr(import_service, "WorkflowService", Mock(return_value=SimpleNamespace(publish_workflow=publish))) - monkeypatch.setattr( - import_service, - "sessionmaker", - lambda _engine: SimpleNamespace(begin=lambda: nullcontext(database.session)), - ) - MigrationImportService()._ensure_workflow_app_is_published( + publisher = create_autospec(WorkflowPublisher, instance=True, spec_set=True) + MigrationImportService( + app_dsl=Mock(), workflows=publisher, workflow_tools=workflow_tools + )._ensure_workflow_app_is_published( ImportTarget("tenant-1", "target", "account-1", "owner@example.com"), account, app_id, session=database.session, ) - - assert database.session.get(App, app_id).workflow_id == "published-workflow" + publisher.publish.assert_called_once() + context, published_app_id = publisher.publish.call_args.args + assert context.account_id == account.id + assert context.active_workspace_id == "tenant-1" + assert published_app_id == app_id @pytest.mark.parametrize("id_strategy", [IdStrategy.PRESERVE_ID, IdStrategy.GENERATE_NEW_ID]) def test_workflow_tool_import_id_follows_id_strategy( - monkeypatch: pytest.MonkeyPatch, database: Database, id_strategy: IdStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, id_strategy: IdStrategy, *, workflow_tools ): created_kwargs = [] _persist_tenant_account(database.session) @@ -536,8 +556,10 @@ def create_workflow_tool(**kwargs) -> None: app_id=kwargs["workflow_app_id"], ) - monkeypatch.setattr(import_service.WorkflowToolManageService, "create_workflow_tool", create_workflow_tool) - StrategyImportService()._import_workflow_tools( + monkeypatch.setattr(workflow_tools, "create_workflow_tool", create_workflow_tool) + StrategyImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -563,7 +585,7 @@ def create_workflow_tool(**kwargs) -> None: ] -def test_workflow_tool_skip_records_id_mapping(database: Database): +def test_workflow_tool_skip_records_id_mapping(database: Database, *, workflow_tools): _persist_tenant_account(database.session) source_app_id = "00000000-0000-0000-0000-000000000031" target_app_id = "00000000-0000-0000-0000-000000000032" @@ -578,7 +600,9 @@ class SkipImportService(MigrationImportService): def _ensure_workflow_app_is_published(self, target, account, app_id, session): return None - SkipImportService()._import_workflow_tools( + SkipImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_workflow_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -600,7 +624,7 @@ def _ensure_workflow_app_is_published(self, target, account, app_id, session): @pytest.mark.parametrize("conflict_strategy", [ConflictStrategy.SKIP, ConflictStrategy.UPDATE]) def test_api_tool_existing_provider_records_id_mapping( - monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy, *, workflow_tools ): _persist_api_provider(database.session, provider_id="target-api-provider-id") _persist_api_provider(database.session, provider_id="other-tenant-provider-id", tenant_id="tenant-2") @@ -613,7 +637,9 @@ def test_api_tool_existing_provider_records_id_mapping( ) monkeypatch.setattr(import_service.ApiToolManageService, "update_api_tool_provider", lambda **kwargs: None) - MigrationImportService()._import_api_tools( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_api_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -644,7 +670,7 @@ def test_api_tool_existing_provider_records_id_mapping( ) -def test_api_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_api_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools): id_mapping = {} monkeypatch.setattr( @@ -656,7 +682,9 @@ def create_api_tool_provider(**_kwargs) -> None: monkeypatch.setattr(import_service.ApiToolManageService, "create_api_tool_provider", create_api_tool_provider) - MigrationImportService()._import_api_tools( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_api_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -680,7 +708,9 @@ def create_api_tool_provider(**_kwargs) -> None: assert id_mapping["source-api-provider-id"] == "target-api-provider-id" -def test_mcp_tool_import_restores_exported_tool_list(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_mcp_tool_import_restores_exported_tool_list( + monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools +): provider = _persist_mcp_provider(database.session, provider_id="target-provider-id") report_items = [] @@ -693,7 +723,9 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -729,7 +761,7 @@ def update_provider(self, **kwargs): @pytest.mark.parametrize("conflict_strategy", [ConflictStrategy.SKIP, ConflictStrategy.UPDATE]) def test_mcp_tool_existing_provider_records_id_mapping( - monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy + monkeypatch: pytest.MonkeyPatch, database: Database, conflict_strategy: ConflictStrategy, *, workflow_tools ): _persist_mcp_provider(database.session, provider_id="target-mcp-provider-id") _persist_mcp_provider( @@ -751,7 +783,9 @@ def update_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -787,7 +821,7 @@ def update_provider(self, **kwargs): ] -def test_mcp_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database): +def test_mcp_tool_create_records_id_mapping(monkeypatch: pytest.MonkeyPatch, database: Database, *, workflow_tools): id_mapping = {} class StubMCPToolManageService: @@ -805,7 +839,9 @@ def create_provider(self, **kwargs): monkeypatch.setattr(import_service, "MCPToolManageService", StubMCPToolManageService) - MigrationImportService()._import_mcp_tools( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._import_mcp_tools( MigrationPackage.from_mapping( { "metadata": {"version": "1", "source_scope": "single"}, @@ -836,7 +872,9 @@ def create_provider(self, **kwargs): assert id_mapping["source-mcp-provider-id"] == "target-mcp-provider-id" -def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_workflow_context(database: Database): +def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_workflow_context( + database: Database, *, workflow_tools +): report_items = [] package = MigrationPackage.from_mapping( { @@ -875,7 +913,9 @@ def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_work } ) - MigrationImportService()._preflight_dependency_only_mcp( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._preflight_dependency_only_mcp( package, ImportTarget( tenant_id="tenant-1", @@ -899,7 +939,9 @@ def test_dependency_only_mcp_preflight_reports_missing_target_provider_with_work ] -def test_dependency_only_mcp_lookup_does_not_compare_non_uuid_identifier_to_uuid_id(database: Database): +def test_dependency_only_mcp_lookup_does_not_compare_non_uuid_identifier_to_uuid_id( + database: Database, *, workflow_tools +): provider = _persist_mcp_provider(database.session, provider_id="provider-1", server_identifier="my-test-mcp-server") statements = [] @@ -908,7 +950,9 @@ def capture_statement(orm_execute_state) -> None: event.listen(database.session, "do_orm_execute", capture_statement) try: - result = MigrationImportService()._find_dependency_only_mcp_provider( + result = MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._find_dependency_only_mcp_provider( "tenant-1", "my-test-mcp-server", "my-test-mcp", @@ -922,7 +966,7 @@ def capture_statement(orm_execute_state) -> None: assert f"{MCPToolProvider.__tablename__}.id" not in where_clause -def test_dependency_only_mcp_preflight_reports_available_target_provider(database: Database): +def test_dependency_only_mcp_preflight_reports_available_target_provider(database: Database, *, workflow_tools): report_items = [] package = MigrationPackage.from_mapping( { @@ -943,7 +987,9 @@ def test_dependency_only_mcp_preflight_reports_available_target_provider(databas tenant_id="tenant-2", ) - MigrationImportService()._preflight_dependency_only_mcp( + MigrationImportService( + app_dsl=Mock(), workflows=_workflow_publisher(), workflow_tools=workflow_tools + )._preflight_dependency_only_mcp( package, ImportTarget( tenant_id="tenant-1", @@ -966,7 +1012,7 @@ def test_dependency_only_mcp_preflight_reports_available_target_provider(databas ] -def test_import_package_imports_workflow_tool_provider_apps_before_consumers(database: Database): +def test_import_package_imports_workflow_tool_provider_apps_before_consumers(database: Database, *, workflow_tools): events = [] class StubResolver(ImportTargetResolver): @@ -1041,9 +1087,12 @@ def _import_mcp_tools( } ) - OrderedImportService(target_resolver=StubResolver()).import_package( - ImportRequest(package=package), session=database.session - ) + OrderedImportService( + app_dsl=Mock(), + workflows=_workflow_publisher(), + target_resolver=StubResolver(), + workflow_tools=workflow_tools, + ).import_package(ImportRequest(package=package), session=database.session) assert events == [ ("api_tools", "imported"), diff --git a/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py b/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py index 13c51405501a59..90bee38bd4960f 100644 --- a/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py +++ b/api/tests/unit_tests/services/human_input_v2/test_form_delivery_runtime.py @@ -237,11 +237,11 @@ def test_factory_runs_v2_and_reuses_persisted_approval( ) -> None: from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context from core.db.session_factory import session_factory - from core.workflow.node_factory import DifyNodeFactory from core.workflow.nodes.human_input_v2.events import NodeRunHumanInputV2FormRequiredEvent from core.workflow.system_variables import SystemVariableKey, system_variable_selector from graphon.engine_events import NodeRunPauseRequestedEvent from graphon.runtime import InitParams, RuntimeState + from services.workflow.execution.adapters.node_factory import DifyNodeFactory monkeypatch.setattr(session_factory, "create_session", context.sessions) pool = VariablePool() @@ -1638,6 +1638,7 @@ def reject_live_reload() -> Session: workflow_runs=repository, node_executions=DifyAPISQLAlchemyWorkflowNodeExecutionRepository(context.sessions), session_factory=context.sessions, + tool_providers=tool_providers, ) from configs import dify_config from extensions.ext_application_services import build_application_services @@ -1747,7 +1748,7 @@ def test_pause_snapshot_resolves_mixed_versions_in_checkpoint_order( @pytest.mark.parametrize("initiator", [False, True]) @pytest.mark.usefixtures("providers") def test_console_pause_details_follow_checkpoint_with_mixed_form_versions( - context: Context, monkeypatch: pytest.MonkeyPatch, initiator: bool + context: Context, monkeypatch: pytest.MonkeyPatch, initiator: bool, *, tool_providers: ToolProviderIcons ) -> None: from core.app.layers.pause_state_persist_layer import WorkflowResumptionContext from core.workflow.nodes.human_input.pause_reason import HumanInputRequired @@ -1823,6 +1824,7 @@ def test_console_pause_details_follow_checkpoint_with_mixed_form_versions( workflow_runs=repository, node_executions=DifyAPISQLAlchemyWorkflowNodeExecutionRepository(context.sessions), session_factory=context.sessions, + tool_providers=tool_providers, ) request = RequestContext(request_id="request", trace_id="trace", account_id=_ACCOUNT, active_workspace_id=_TENANT) @@ -1848,7 +1850,7 @@ def test_console_pause_details_follow_checkpoint_with_mixed_form_versions( @pytest.mark.parametrize("checkpoint", ["invalid", "missing_form"]) def test_console_pause_details_do_not_fall_back_when_checkpoint_is_invalid( - context: Context, monkeypatch: pytest.MonkeyPatch, checkpoint: str + context: Context, monkeypatch: pytest.MonkeyPatch, checkpoint: str, *, tool_providers: ToolProviderIcons ) -> None: from machinery.context import RequestContext from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository @@ -1863,6 +1865,7 @@ def test_console_pause_details_do_not_fall_back_when_checkpoint_is_invalid( workflow_runs=repository, node_executions=DifyAPISQLAlchemyWorkflowNodeExecutionRepository(context.sessions), session_factory=context.sessions, + tool_providers=tool_providers, ) request = RequestContext(request_id="request", trace_id="trace", account_id=_ACCOUNT, active_workspace_id=_TENANT) @@ -3049,7 +3052,6 @@ def test_initial_delivery_results_control_graph_failure_and_approval( ) -> None: from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, build_dify_run_context from core.db.session_factory import session_factory - from core.workflow.node_factory import DifyNodeFactory from core.workflow.system_variables import SystemVariableKey, system_variable_selector from graphon.engine import Engine from graphon.engine.command import InMemoryChannel @@ -3068,6 +3070,7 @@ def test_initial_delivery_results_control_graph_failure_and_approval( from graphon.runtime import InitParams, RuntimeState from repositories.human_input_v2.delivery_attempt_repository import DeliveryStatus from repositories.human_input_v2.email_channel.ports import EmailProviderOperationError + from services.workflow.execution.adapters.node_factory import DifyNodeFactory monkeypatch.setattr(session_factory, "create_session", context.sessions) provider_accept = providers[1].on_send diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_service.py index f4bd9699b25bb9..78185c8ce77f14 100644 --- a/api/tests/unit_tests/services/knowledge/test_dataset_service.py +++ b/api/tests/unit_tests/services/knowledge/test_dataset_service.py @@ -54,6 +54,7 @@ from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus, ProcessRuleMode, SegmentStatus from models.model import UploadFile from models.source import DataSourceOauthBinding +from repositories.knowledge.collection_binding_repository import DatasetCollectionBindingRepository from repositories.knowledge.segment_repository import SQLAlchemySegmentRepository from services.enterprise import rbac_service from services.entities.knowledge_entities.rag_pipeline_entities import ( @@ -62,8 +63,9 @@ ) from services.errors.base import NoPermissionError from services.errors.dataset import DatasetNameDuplicateError +from services.errors.document import DocumentNotFoundError from services.knowledge import dataset_service -from services.knowledge.dataset_service import DatasetCollectionBindingService, DatasetPermissionService, SegmentService +from services.knowledge.dataset_service import DatasetPermissionService, SegmentService from services.knowledge.dataset_service import DatasetService as _dataset_service_dataset_DatasetService from services.knowledge.dataset_service import DocumentService as _dataset_service_lock_not_owned_DocumentService from services.knowledge.dataset_service import DocumentService as _metadata_built_in_fields_DocumentService @@ -88,7 +90,6 @@ DocumentIndexingError, FileInfo, FileNotExistsError, - NotFound, NotionIcon, NotionInfo, NotionPage, @@ -102,7 +103,6 @@ TenantAccountRole, WebsiteInfo, _make_features, - _make_knowledge_configuration, _make_lock_context, _make_retrieval_model, _make_upload_knowledge_config, @@ -1112,8 +1112,10 @@ def test_update_pipeline_node_data_rolls_back_real_session_on_failure(self, sqli lambda _session, _previous_transaction: transaction_events.append("rollback"), ) - with _dataset_service_dataset_patch("services.knowledge.dataset_service.RagPipelineService") as service_cls: - service_cls.return_value.get_published_workflow.side_effect = RuntimeError("boom") + with _dataset_service_dataset_patch( + "services.knowledge.dataset_service.WorkflowDefinitionStore" + ) as service_cls: + service_cls.get_published_workflow.side_effect = RuntimeError("boom") with pytest.raises(RuntimeError, match="boom"): _dataset_service_dataset_DatasetService._update_pipeline_knowledge_base_node_data( dataset, "user-1", sqlite_session @@ -1258,7 +1260,7 @@ def test_apply_new_embedding_settings_uses_real_collection_binding(self, unbound _dataset_service_dataset_patch("services.knowledge.dataset_service.current_user", account), _dataset_service_dataset_patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, _dataset_service_dataset_patch.object( - DatasetCollectionBindingService, + DatasetCollectionBindingRepository, "get_dataset_collection_binding", return_value=collection_binding, ), @@ -1325,162 +1327,6 @@ def test_check_summary_index_setting_model_changed( is expected ) - class TestDatasetServiceRagPipelineSettings: - def test_requires_current_tenant(self, unbound_session: Session) -> None: - account = _dataset_service_dataset_account() - account._current_tenant = None - - with ( - _dataset_service_dataset_patch("services.knowledge.dataset_service.current_user", account), - pytest.raises(ValueError, match="Current user or current tenant not found"), - ): - _dataset_service_dataset_DatasetService.update_rag_pipeline_dataset_settings( - _dataset_service_dataset_dataset(), - _make_knowledge_configuration(), - session=unbound_session, - ) - - def test_unpublished_high_quality_settings_use_real_merged_dataset(self, sqlite_session: Session) -> None: - dataset = _dataset_service_dataset_dataset(indexing_technique=IndexTechniqueType.ECONOMY) - sqlite_session.add(dataset) - sqlite_session.commit() - account = _dataset_service_dataset_account() - embedding_model = SimpleNamespace(provider="provider-2", model_name="embedding-2") - collection_binding = DatasetCollectionBinding( - provider_name="provider-2", - model_name="embedding-2", - type="dataset", - collection_name="collection", - ) - collection_binding.id = "collection-binding-2" - - with ( - _dataset_service_dataset_patch("services.knowledge.dataset_service.current_user", account), - _dataset_service_dataset_patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, - _dataset_service_dataset_patch.object( - _dataset_service_dataset_DatasetService, "check_is_multimodal_model", return_value=True - ), - _dataset_service_dataset_patch.object( - DatasetCollectionBindingService, - "get_dataset_collection_binding", - return_value=collection_binding, - ), - ): - model_manager_cls.for_tenant.return_value.get_model_instance.return_value = embedding_model - _dataset_service_dataset_DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - embedding_model_provider="provider-2", - embedding_model="embedding-2", - summary_index_setting={"enable": True}, - ), - session=sqlite_session, - ) - - persisted = sqlite_session.get(Dataset, dataset.id) - assert persisted is not None - assert persisted.indexing_technique == IndexTechniqueType.HIGH_QUALITY - assert persisted.embedding_model == "embedding-2" - assert persisted.collection_binding_id == collection_binding.id - assert persisted.is_multimodal is True - - def test_unpublished_economy_settings_update_keyword_number(self, sqlite_session: Session) -> None: - dataset = _dataset_service_dataset_dataset(indexing_technique=IndexTechniqueType.HIGH_QUALITY) - sqlite_session.add(dataset) - sqlite_session.commit() - - with _dataset_service_dataset_patch( - "services.knowledge.dataset_service.current_user", _dataset_service_dataset_account() - ): - _dataset_service_dataset_DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - indexing_technique=IndexTechniqueType.ECONOMY, - keyword_number=17, - ), - session=sqlite_session, - ) - - persisted = sqlite_session.get(Dataset, dataset.id) - assert persisted is not None - assert persisted.indexing_technique == IndexTechniqueType.ECONOMY - assert persisted.keyword_number == 17 - - def test_published_economy_settings_commit_keyword_change_and_dispatch_no_task( - self, sqlite_session: Session - ) -> None: - dataset = _dataset_service_dataset_dataset( - indexing_technique=IndexTechniqueType.ECONOMY, chunk_structure="paragraph" - ) - dataset.keyword_number = 4 - sqlite_session.add(dataset) - sqlite_session.commit() - - with ( - _dataset_service_dataset_patch( - "services.knowledge.dataset_service.current_user", _dataset_service_dataset_account() - ), - _dataset_service_dataset_patch( - "services.knowledge.dataset_service.deal_dataset_index_update_task.delay" - ) as update_task, - ): - _dataset_service_dataset_DatasetService.update_rag_pipeline_dataset_settings( - dataset, - _make_knowledge_configuration( - chunk_structure="paragraph", - indexing_technique=IndexTechniqueType.ECONOMY, - keyword_number=12, - ), - has_published=True, - session=sqlite_session, - ) - - assert sqlite_session.get(Dataset, dataset.id).keyword_number == 12 - update_task.assert_not_called() - - @pytest.mark.parametrize( - ("dataset", "configuration", "message"), - [ - ( - _dataset_service_dataset_dataset(chunk_structure="paragraph"), - _make_knowledge_configuration(chunk_structure="sentence"), - "Chunk structure is not allowed", - ), - ( - _dataset_service_dataset_dataset( - indexing_technique=IndexTechniqueType.HIGH_QUALITY, chunk_structure="paragraph" - ), - _make_knowledge_configuration( - chunk_structure="paragraph", - indexing_technique=IndexTechniqueType.ECONOMY, - ), - "not allowed to be updated to economy", - ), - ], - ) - def test_published_settings_reject_incompatible_changes( - self, - sqlite_session: Session, - dataset: Dataset, - configuration, - message: str, - ) -> None: - sqlite_session.add(dataset) - sqlite_session.commit() - - with ( - _dataset_service_dataset_patch( - "services.knowledge.dataset_service.current_user", _dataset_service_dataset_account() - ), - pytest.raises(ValueError, match=message), - ): - _dataset_service_dataset_DatasetService.update_rag_pipeline_dataset_settings( - dataset, - configuration, - has_published=True, - session=sqlite_session, - ) - class TestDatasetPermissions: def test_check_dataset_permission_enforces_tenant_and_partial_members(self, sqlite_session: Session) -> None: dataset = _dataset_service_dataset_dataset( @@ -1974,23 +1820,33 @@ def test_delete_document_dispatches_cleanup_after_the_deletion_commits(self, sql sqlite_session.commit() visible_when_dispatched: list[bool] = [] - def record_visibility(*_args: object) -> None: + dispatched: list[tuple[str, dict[str, object]]] = [] + + def record_visibility(sender: str, **kwargs: object) -> None: # Cleanup and in-flight indexing run in other sessions; they must not see the row. with Session(sqlite_session.get_bind()) as other_session: visible_when_dispatched.append( other_session.get(_dataset_service_document_Document, "doc-1") is not None ) + dispatched.append((sender, kwargs)) - with _dataset_service_document_patch( - "events.event_handlers.clean_when_document_deleted.clean_document_task" - ) as clean_task: - clean_task.delay.side_effect = record_visibility + dataset_service.document_was_deleted.connect(record_visibility, weak=False) + try: _dataset_service_document_DocumentService.delete_document(document, sqlite_session) + finally: + dataset_service.document_was_deleted.disconnect(record_visibility) - clean_task.delay.assert_called_once_with( - "doc-1", dataset.id, _dataset_service_document_IndexStructureType.PARAGRAPH_INDEX, "file-1" - ) assert visible_when_dispatched == [False] + assert dispatched == [ + ( + "doc-1", + { + "dataset_id": dataset.id, + "doc_form": _dataset_service_document_IndexStructureType.PARAGRAPH_INDEX, + "file_id": "file-1", + }, + ) + ] def test_rename_document_raises_when_dataset_is_missing(self, sqlite_session: Session): with _dataset_service_document_patch( @@ -2322,7 +2178,7 @@ def test_save_document_without_dataset_id_creates_high_quality_dataset_with_defa "services.knowledge.dataset_service.FeatureService.get_features", return_value=_make_features() ), _dataset_service_document_patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ), _dataset_service_document_patch.object( @@ -2463,7 +2319,7 @@ def test_update_document_with_dataset_id_raises_when_document_is_missing( with _dataset_service_document_patch.object( _dataset_service_document_DatasetService, "check_dataset_model_setting" ) as check_model_setting: - with pytest.raises(NotFound, match="Document not found"): + with pytest.raises(DocumentNotFoundError, match="Document not found"): _dataset_service_document_DocumentService.update_document_with_dataset_id( dataset, document_data, @@ -3653,7 +3509,7 @@ def test_save_document_with_dataset_id_initializes_high_quality_dataset_from_def ), _dataset_service_document_patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, _dataset_service_document_patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ) as get_binding, _dataset_service_document_patch.object( @@ -3726,7 +3582,7 @@ def test_save_document_with_dataset_id_uses_explicit_embedding_and_retrieval_mod ), _dataset_service_document_patch("services.knowledge.dataset_service.ModelManager") as model_manager_cls, _dataset_service_document_patch( - "services.knowledge.dataset_service.DatasetCollectionBindingService.get_dataset_collection_binding", + "services.knowledge.dataset_service.DatasetCollectionBindingRepository.get_dataset_collection_binding", return_value=binding, ) as get_binding, _dataset_service_document_patch.object( diff --git a/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py new file mode 100644 index 00000000000000..51c4a381b41f52 --- /dev/null +++ b/api/tests/unit_tests/services/knowledge/test_runner_retrieval_boundary.py @@ -0,0 +1,352 @@ +"""Chat and Completion release retrieval reads before model and HTTP calls.""" + +import json +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, NoReturn, Self, override +from uuid import uuid4 + +import httpx +import pytest +from flask import Flask +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import QueuePool + +from core.app.app_config.entities import DatasetEntity, DatasetRetrieveConfigEntity, ModelConfig +from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom +from core.app.apps.chat.app_config_manager import ChatAppConfig +from core.app.apps.completion.app_config_manager import CompletionAppConfig +from core.app.entities.app_invoke_entities import ( + ChatAppGenerateEntity, + CompletionAppGenerateEntity, + InvokeFrom, + ModelConfigWithCredentialsEntity, +) +from core.app.entities.queue_entities import AppQueueEvent +from core.db import session_factory +from core.model_manager import ModelManager +from core.tools.entities.tool_entities import ToolInvokeMessage +from extensions.application_services.retrieval import build_dataset_retrieval +from graphon.model_runtime.entities.llm_entities import LLMResultChunk, LLMResultChunkDelta, LLMUsage +from graphon.model_runtime.entities.message_entities import AssistantPromptMessage +from graphon.model_runtime.entities.model_entities import ModelFeature +from models.annotation_reply import AnnotationReply +from models.base import TypeBase +from models.dataset import ( + Dataset, + DatasetMetadata, + DatasetQuery, + Document, + ExternalKnowledgeApis, + ExternalKnowledgeBindings, +) +from models.enums import ConversationFromSource +from models.model import App, AppMode, Conversation, Message +from repositories.app.generation_repository import AppGenerationRepository +from repositories.knowledge.retrieval_repository import KnowledgeRetrievalRepository +from services.app.generation.adapters.chat_runner import ChatAppRunner +from services.app.generation.adapters.completion_runner import CompletionAppRunner +from services.entities.external_knowledge_entities.external_knowledge_entities import ExternalKnowledgeApiSetting +from services.knowledge.external.service import ExternalDatasetService +from services.knowledge.retrieval.adapters.resource_events import DatasetIndexToolCallbackHandler +from services.knowledge.retrieval.dataset_retrieval import DatasetRetrieval +from services.tools.dataset.tool import DatasetRetrieverTool +from tests.helpers.unit.model_factories import make_dataset + + +@dataclass +class Queue(AppQueueManager): + events: list[AppQueueEvent] = field(default_factory=list) + + @override + def publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None: + del pub_from + self.events.append(event) + + @override + def _publish(self, event: AppQueueEvent, pub_from: PublishFrom) -> None: + del pub_from + self.events.append(event) + + +class NoAnnotations: + def query( + self, + *, + tenant_id: str, + app_id: str, + message_id: str, + query: str, + user_id: str, + from_source: ConversationFromSource, + ) -> AnnotationReply | None: + del tenant_id, app_id, message_id, query, user_id, from_source + return None + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION, "agent-tool"]) +@pytest.mark.parametrize("strategy", ["single", "multiple"]) +def test_runner_releases_connections_during_metadata_and_external_retrieval( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mode: AppMode | Literal["agent-tool"], + strategy: Literal["single", "multiple"], +) -> None: + engine = create_engine( + f"sqlite:///{tmp_path / 'retrieval.db'}", + poolclass=QueuePool, + pool_size=1, + max_overflow=0, + ) + pool = engine.pool + assert isinstance(pool, QueuePool) + sessions = sessionmaker(engine) # Production default: expire_on_commit=True. + tables = [ + TypeBase.metadata.tables[model.__tablename__] + for model in ( + Dataset, + Document, + DatasetMetadata, + DatasetQuery, + ExternalKnowledgeApis, + ExternalKnowledgeBindings, + ) + ] + TypeBase.metadata.create_all( + engine, + tables=tables, + ) + tenant_id, dataset_id, user_id, app_id = [str(uuid4()) for _ in range(4)] + with sessions.begin() as session: + dataset = make_dataset( + dataset_id=dataset_id, + tenant_id=tenant_id, + created_by=user_id, + provider="external", + indexing_technique="high_quality", + ) + dataset.retrieval_model = {"top_k": 3} + api = ExternalKnowledgeApis( + name="External", + description="", + tenant_id=tenant_id, + created_by=user_id, + settings=json.dumps({"endpoint": "https://knowledge.example", "api_key": "secret"}), + updated_by=None, + ) + session.add_all([dataset, api]) + session.flush() + session.add_all( + [ + ExternalKnowledgeBindings( + tenant_id=tenant_id, + dataset_id=dataset_id, + external_knowledge_api_id=api.id, + external_knowledge_id="collection", + created_by=user_id, + ), + DatasetMetadata( + tenant_id=tenant_id, dataset_id=dataset_id, name="category", type="string", created_by=user_id + ), + ] + ) + calls: list[str] = [] + + def no_global_database(*_args: object, **_kwargs: object) -> NoReturn: + raise AssertionError("retrieval used the global database") + + monkeypatch.setattr(session_factory, "create_session", no_global_database) + + @dataclass + class Schema: + features: list[ModelFeature] = field(default_factory=list) + + class Model: + model_name = "model" + credentials: dict[str, object] = {} + provider_model_bundle = object() + + @property + def model_type_instance(self) -> Self: + return self + + def get_model_schema(self, **_kwargs: object) -> Schema: + return Schema() + + def get_model_instance(self, **_kwargs: object) -> Self: + return self + + def invoke_llm(self, **_kwargs: object) -> Iterator[LLMResultChunk]: + assert pool.checkedout() == 0 + calls.append("metadata-model") + yield LLMResultChunk( + model="model", + delta=LLMResultChunkDelta( + index=0, + message=AssistantPromptMessage( + content=json.dumps( + { + "metadata_map": [ + { + "metadata_field_name": "category", + "metadata_field_value": "docs", + "comparison_operator": "is", + } + ] + } + ) + ), + usage=LLMUsage.empty_usage(), + ), + ) + + model = Model() + model_config = ModelConfigWithCredentialsEntity.model_construct( + provider="provider", + model="model", + mode="chat", + provider_model_bundle=model.provider_model_bundle, + credentials={}, + parameters={}, + stop=[], + model_schema=Schema(), + ) + monkeypatch.setattr(ModelManager, "for_tenant", lambda **_kwargs: model) + monkeypatch.setattr(DatasetRetrieval, "_fetch_model_config", lambda *_args, **_kwargs: (model, model_config)) + + def metadata_prompt(_self: DatasetRetrieval, **kwargs: object) -> tuple[list[object], list[object]]: + assert kwargs["metadata_fields"] == ["category"] # A real database read happened before model I/O. + return [], [] + + monkeypatch.setattr(DatasetRetrieval, "_get_prompt_template", metadata_prompt) + + class Router: + def __init__(self, **_kwargs: object) -> None: + pass + + def invoke(self, *_args: object, **_kwargs: object) -> tuple[str, LLMUsage]: + assert pool.checkedout() == 0 + calls.append("router") + return dataset_id, LLMUsage.empty_usage() + + monkeypatch.setattr("services.knowledge.retrieval.dataset_retrieval.ReactMultiDatasetRouter", Router) + + def external_request(request: ExternalKnowledgeApiSetting, _files: dict[str, object] | None) -> httpx.Response: + assert pool.checkedout() == 0 + assert request.url == "https://knowledge.example/retrieval" + assert request.headers is not None + assert request.params is not None + assert request.headers["Authorization"] == "Bearer secret" + metadata_condition = request.params["metadata_condition"] + assert isinstance(metadata_condition, Mapping) + conditions = metadata_condition["conditions"] + assert isinstance(conditions, list) + condition = conditions[0] + assert isinstance(condition, Mapping) + assert condition["value"] == "docs" + calls.append("external-knowledge") + return httpx.Response( + 200, json={"records": [{"content": "retrieved context", "title": "Docs", "score": 0.9, "metadata": {}}]} + ) + + monkeypatch.setattr(ExternalDatasetService, "process_external_api", external_request) + config_type = ChatAppConfig if mode == AppMode.CHAT else CompletionAppConfig + config = config_type.model_construct( + tenant_id=tenant_id, + app_id=app_id, + app_mode=mode, + app_model_config_dict={}, + prompt_template=None, + dataset=DatasetEntity( + dataset_ids=[dataset_id], + retrieve_config=DatasetRetrieveConfigEntity( + retrieve_strategy=strategy, + top_k=3, + reranking_enabled=False, + metadata_filtering_mode="automatic", + metadata_model_config=ModelConfig(provider="provider", name="model", mode="chat"), + ), + ), + ) + entity_type = ChatAppGenerateEntity if mode == AppMode.CHAT else CompletionAppGenerateEntity + entity = entity_type.model_construct( + task_id="task", + app_config=config, + model_conf=model_config, + inputs={}, + files=[], + user_id=user_id, + query="question", + invoke_from=InvokeFrom.DEBUGGER, + stream=False, + ) + runner_type = ChatAppRunner if mode == AppMode.CHAT else CompletionAppRunner + runner = runner_type(records=AppGenerationRepository(sessions), retrieval=build_dataset_retrieval(sessions)) + prompts: list[str | None] = [] + + def organize_prompt(**kwargs: object) -> tuple[list[object], list[object]]: + context = kwargs.get("context") + assert context is None or isinstance(context, str) + prompts.append(context) + return [], [] + + monkeypatch.setattr(runner, "organize_prompt_messages", organize_prompt) + monkeypatch.setattr(runner, "moderation_for_inputs", lambda **_kwargs: (None, {}, "question")) + # Stop after retrieval and prompt preparation; the assertions cover all retrieval I/O. + monkeypatch.setattr(runner, "check_hosting_moderation", lambda **_kwargs: True) + queue = Queue() + app_mode = mode if isinstance(mode, AppMode) else AppMode.AGENT_CHAT + app = App(id=app_id, tenant_id=tenant_id, mode=app_mode) + message = Message(id=str(uuid4())) + try: + with Flask(__name__).app_context(): + if mode == "agent-tool": + assert config.dataset is not None + tools = DatasetRetrieverTool.get_dataset_tools( + records=KnowledgeRetrievalRepository(sessions), + retrieval=build_dataset_retrieval(sessions), + tenant_id=tenant_id, + app_id=app_id, + dataset_ids=[dataset_id], + retrieve_config=config.dataset.retrieve_config, + return_resource=True, + invoke_from=InvokeFrom.DEBUGGER, + hit_callback=DatasetIndexToolCallbackHandler(queue), + user_id=user_id, + inputs={}, + ) + with sessions() as tool_session: + tool_message = list( + tools[0].invoke( + session=tool_session, + user_id=user_id, + tool_parameters={"query": "question"}, + ) + )[0] + assert isinstance(tool_message.message, ToolInvokeMessage.TextMessage) + prompts.append(tool_message.message.text) + elif mode == AppMode.CHAT: + assert isinstance(runner, ChatAppRunner) + assert isinstance(entity, ChatAppGenerateEntity) + runner.run(entity, queue, Conversation(), message, app, NoAnnotations()) + else: + assert isinstance(runner, CompletionAppRunner) + assert isinstance(entity, CompletionAppGenerateEntity) + runner.run(entity, queue, message, app) + assert calls == [ + "metadata-model", + *(["router"] if strategy == "single" and mode != "agent-tool" else []), + "external-knowledge", + ] + assert prompts[-1] == "retrieved context" + assert len(queue.events) == 1 + with sessions() as session: + audit = session.scalar(select(DatasetQuery)) + assert audit is not None + assert audit.dataset_id == dataset_id + assert audit.source_app_id == app_id + assert pool.checkedout() == 0 + finally: + engine.dispose() diff --git a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py index bf631d0c8ac664..37064c738cbab2 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_pipeline_generate_service.py @@ -1,18 +1,17 @@ -from collections.abc import Callable - import pytest from pytest_mock import MockerFixture from sqlalchemy.orm import Session -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.rag.index_processor.constant.index_type import IndexStructureType from models.dataset import Dataset, Document, Pipeline from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus -from models.model import Account, App, AppMode, EndUser +from models.model import Account, EndUser from models.workflow import Workflow, WorkflowType from services.knowledge.resource_scope import DatasetRef from services.rag_pipeline.pipeline_generate_service import PipelineGenerateService +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator @pytest.fixture @@ -32,17 +31,6 @@ def _make_account(*, account_id: str = "user-1") -> Account: return account -def _make_app(*, max_active_requests: int) -> App: - return App( - tenant_id="tenant-1", - name="Pipeline App", - mode=AppMode.RAG_PIPELINE, - enable_site=False, - enable_api=False, - max_active_requests=max_active_requests, - ) - - def _make_workflow(*, workflow_id: str = "wf-1") -> Workflow: return Workflow( id=workflow_id, @@ -84,56 +72,7 @@ def _make_document( ) -def test_get_max_active_requests_uses_smallest_non_zero_limit( - config_overrides: Callable[..., None], -) -> None: - config_overrides(APP_DEFAULT_ACTIVE_REQUESTS=5, APP_MAX_ACTIVE_REQUESTS=3) - - app_model = _make_app(max_active_requests=10) - - result = PipelineGenerateService._get_max_active_requests(app_model) - - assert result == 3 - - -def test_get_max_active_requests_returns_zero_when_all_unlimited( - config_overrides: Callable[..., None], -) -> None: - config_overrides(APP_DEFAULT_ACTIVE_REQUESTS=0, APP_MAX_ACTIVE_REQUESTS=0) - - app_model = _make_app(max_active_requests=0) - - result = PipelineGenerateService._get_max_active_requests(app_model) - - assert result == 0 - - -@pytest.mark.parametrize( - ("invoke_from", "workflow", "expected_error"), - [ - (InvokeFrom.DEBUGGER, None, "Workflow not initialized"), - (InvokeFrom.WEB_APP, None, "Workflow not published"), - (InvokeFrom.DEBUGGER, _make_workflow(), None), - ], -) -def test_get_workflow(mocker: MockerFixture, invoke_from, workflow, expected_error, sqlite_session: Session) -> None: - rag_pipeline_service_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.RagPipelineService") - rag_pipeline_service = rag_pipeline_service_cls.return_value - rag_pipeline_service.get_draft_workflow.return_value = workflow - rag_pipeline_service.get_published_workflow.return_value = workflow - - pipeline = _make_pipeline() - session = sqlite_session - - if expected_error: - with pytest.raises(ValueError, match=expected_error): - PipelineGenerateService._get_workflow(pipeline, invoke_from, session) - else: - result = PipelineGenerateService._get_workflow(pipeline, invoke_from, session) - assert result == workflow - - -def test_generate_updates_document_status_and_returns_event_stream( +def test_generate_delegates_original_document_and_returns_event_stream( mocker: MockerFixture, sqlite_session: Session, pipeline_generator ) -> None: dataset = _make_dataset() @@ -143,13 +82,14 @@ def test_generate_updates_document_status_and_returns_event_stream( user: Account | EndUser = _make_account() args = {"original_document_id": "doc-1", "query": "hello"} - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - update_status_mock = mocker.patch.object(PipelineGenerateService, "update_document_status") + pipeline_generator.load_workflow.return_value = _make_workflow() + update_status = mocker.patch.object(PipelineGenerateService, "update_document_status") - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") generator_instance = pipeline_generator generator_instance.generate.return_value = "raw-events" - generator_cls.convert_to_event_stream.return_value = "stream-events" + mocker.patch( + "services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-events" + ) result = PipelineGenerateService.generate( pipeline=pipeline, @@ -162,12 +102,13 @@ def test_generate_updates_document_status_and_returns_event_stream( ) assert result == "stream-events" - document_ref = update_status_mock.call_args.args[0] + document_ref = update_status.call_args.args[0] assert document_ref.dataset.tenant_id == "tenant-1" assert document_ref.dataset.dataset_id == "dataset-1" assert document_ref.document_id == "doc-1" - update_status_mock.assert_called_once_with(document_ref, session=sqlite_session) - assert generator_instance.generate.call_args.kwargs["session"] is sqlite_session + update_status.assert_called_once_with(document_ref, session=sqlite_session) + assert generator_instance.generate.call_args.kwargs["args"] == args + assert "session" not in generator_instance.generate.call_args.kwargs def test_generate_rejects_pipeline_dataset_from_another_tenant( @@ -177,7 +118,7 @@ def test_generate_rejects_pipeline_dataset_from_another_tenant( pipeline = _make_pipeline() sqlite_session.add_all([dataset, pipeline]) sqlite_session.commit() - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) + pipeline_generator.load_workflow.return_value = _make_workflow() update_status_mock = mocker.patch.object(PipelineGenerateService, "update_document_status") with pytest.raises(ValueError, match="Pipeline dataset is required"): @@ -194,7 +135,6 @@ def test_generate_rejects_pipeline_dataset_from_another_tenant( def test_generate_rejects_original_document_outside_pipeline_dataset_before_dispatch( - mocker: MockerFixture, sqlite_session: Session, pipeline_generator, ) -> None: @@ -203,8 +143,7 @@ def test_generate_rejects_original_document_outside_pipeline_dataset_before_disp outside_document = _make_document(document_id="foreign-doc", dataset_id="other-dataset", tenant_id="tenant-2") sqlite_session.add_all([dataset, pipeline, outside_document]) sqlite_session.commit() - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") + pipeline_generator.load_workflow.return_value = _make_workflow() with pytest.raises(ValueError, match="Pipeline document not found"): PipelineGenerateService.generate( @@ -263,50 +202,76 @@ def test_update_document_status_rejects_document_outside_owner( # --- generate_single_iteration --- -def test_generate_single_iteration_delegates( - mocker: MockerFixture, sqlite_session: Session, pipeline_generator -) -> None: - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) - - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") +def test_generate_single_iteration_delegates(mocker: MockerFixture, pipeline_generator) -> None: + pipeline_generator.load_workflow.return_value = _make_workflow() generator_instance = pipeline_generator generator_instance.single_iteration_generate.return_value = "raw-iter" - generator_cls.convert_to_event_stream.return_value = "stream-iter" + mocker.patch("services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-iter") pipeline = _make_pipeline() pipeline.id = "p1" user = _make_account(account_id="u1") - session = sqlite_session result = PipelineGenerateService.generate_single_iteration( - pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator + pipeline, user, "node-1", {"key": "val"}, generator=pipeline_generator ) assert result == "stream-iter" generator_instance.single_iteration_generate.assert_called_once() - assert generator_instance.single_iteration_generate.call_args.kwargs["session"] is session + assert "session" not in generator_instance.single_iteration_generate.call_args.kwargs # --- generate_single_loop --- -def test_generate_single_loop_delegates(mocker: MockerFixture, sqlite_session: Session, pipeline_generator) -> None: - mocker.patch.object(PipelineGenerateService, "_get_workflow", return_value=_make_workflow()) +def test_generate_single_loop_delegates(mocker: MockerFixture, pipeline_generator) -> None: + pipeline_generator.load_workflow.return_value = _make_workflow() - generator_cls = mocker.patch("services.rag_pipeline.pipeline_generate_service.PipelineGenerator") generator_instance = pipeline_generator generator_instance.single_loop_generate.return_value = "raw-loop" - generator_cls.convert_to_event_stream.return_value = "stream-loop" + mocker.patch("services.rag_pipeline.pipeline_generate_service.convert_to_event_stream", return_value="stream-loop") pipeline = _make_pipeline() pipeline.id = "p1" user = _make_account(account_id="u1") - session = sqlite_session result = PipelineGenerateService.generate_single_loop( - pipeline, user, "node-1", {"key": "val"}, session, generator=pipeline_generator + pipeline, user, "node-1", {"key": "val"}, generator=pipeline_generator ) assert result == "stream-loop" generator_instance.single_loop_generate.assert_called_once() - assert generator_instance.single_loop_generate.call_args.kwargs["session"] is session + assert "session" not in generator_instance.single_loop_generate.call_args.kwargs + + +@pytest.mark.parametrize("draft", [True, False]) +def test_pipeline_execution_definition_is_owned_snapshot_before_generation( + sqlite_session_factory, + workflow_application, + draft: bool, +) -> None: + from services.errors.app import WorkflowNotFoundError + + pipeline = _make_pipeline() + workflow = _make_workflow() + if not draft: + workflow.version = "published" + pipeline.workflow_id = workflow.id + with sqlite_session_factory.begin() as session: + session.add_all([pipeline, workflow]) + generator = workflow_application.knowledge.pipeline_generator + invocation = InvokeFrom.DEBUGGER if draft else InvokeFrom.PUBLISHED_PIPELINE + loaded = generator.load_workflow(pipeline, invocation) + assert isinstance(loaded, WorkflowSnapshot) + assert (loaded.id, loaded.tenant_id, loaded.app_id, loaded.version) == ( + workflow.id, + workflow.tenant_id, + workflow.app_id, + workflow.version, + ) + assert loaded.graph_dict == {} + assert loaded.features_dict == {} + assert sqlite_session_factory.kw["bind"].pool.checkedout() == 0 + pipeline.tenant_id = "other-tenant" + with pytest.raises(WorkflowNotFoundError): + generator.load_workflow(pipeline, invocation) diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py index c397af2789b39e..f84ef1bb695a28 100644 --- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py +++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py @@ -3,19 +3,20 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime +from functools import partial from types import SimpleNamespace from unittest import mock from unittest.mock import create_autospec import pytest from pytest_mock import MockerFixture -from sqlalchemy.dialects import postgresql from sqlalchemy.orm import Session, sessionmaker -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.app.entities.app_invoke_entities import InvokeFrom from core.rag.index_processor.constant.index_type import IndexStructureType +from core.repositories.factory import DifyCoreRepositoryFactory from graphon.engine_events import NodeRunFailedEvent +from graphon.entities.workflow_node_execution import WorkflowNodeExecution from graphon.enums import ( BuiltinNodeTypes, ErrorStrategy, @@ -26,6 +27,7 @@ from graphon.nodes.start.entities import StartNodeData from graphon.nodes.start.start_node import StartNode from graphon.runtime import InitParams, RuntimeState, VariablePool +from graphon.variable_loader import VariableLoader from models import Account, Tenant from models.dataset import ( Dataset, @@ -38,13 +40,22 @@ from models.enums import DataSourceType, DocumentCreatedFrom, IndexingStatus from models.workflow import Workflow, WorkflowRun from models.workflow import WorkflowType as ModelWorkflowType +from repositories.factory import DifyAPIRepositoryFactory from repositories.knowledge.dataset_read_repository import get_pipeline_dataset +from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository, workflow_snapshot from services.credentials.query import CredentialQuery from services.data_source.credential_gateway import DatasourceProviderCredentialStore from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, PipelineTemplateInfoEntity from services.errors.rag_pipeline import RagPipelineResourceNotFoundError +from services.rag_pipeline import execution_service as execution_module from services.rag_pipeline import rag_pipeline as rag_pipeline_module +from services.rag_pipeline.execution_service import RagPipelineExecutionService from services.rag_pipeline.rag_pipeline import RagPipelineService +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_ref_service import WorkflowRef from tests.helpers.unit.credential_refresh import build_datasource_provider_service @@ -89,6 +100,7 @@ def test_fetch_recommended_plugin_manifests_returns_data_when_enabled( @dataclass class RagPipelineServiceTestContext: service: RagPipelineService + execution: RagPipelineExecutionService session: Session session_maker: sessionmaker[Session] @@ -274,6 +286,9 @@ def rag_pipeline_service( mocker: MockerFixture, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, + *, + workflow_runtime: WorkflowRuntime, ) -> RagPipelineServiceTestContext: mocker.patch( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository", @@ -283,33 +298,26 @@ def rag_pipeline_service( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_run_repository", return_value=MockRepo(), ) - mocker.patch("services.rag_pipeline.rag_pipeline.db", SimpleNamespace(engine=sqlite_session.get_bind())) service = RagPipelineService(session=sqlite_session, session_maker=sqlite_session_factory) return RagPipelineServiceTestContext( service=service, + execution=RagPipelineExecutionService( + workflows=WorkflowDefinitionRepository(session_factory=sqlite_session_factory), + variables=workflow_variables, + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( + sqlite_session_factory + ), + writer_factory=partial( + DifyCoreRepositoryFactory.create_workflow_node_execution_repository, + session_factory=sqlite_session_factory, + ), + documents=SQLAlchemyDocumentRepository(session_factory=sqlite_session_factory), + runtime=workflow_runtime, + ), session=sqlite_session, session_maker=sqlite_session_factory, ) - @staticmethod - def test_get_published_workflow_by_id_locks_restore_source( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext - ) -> None: - session = rag_pipeline_service.session - workflow = _make_workflow() - workflow.version = "v1" - session.add(workflow) - session.flush() - lookup = mocker.spy(session, "scalar") - service = rag_pipeline_service.service - - result = service.get_published_workflow_by_id(_make_pipeline(), workflow.id) - - stmt = lookup.call_args.args[0] - sql = str(stmt.compile(dialect=postgresql.dialect())) - assert result is workflow - assert "FOR UPDATE" in sql - @staticmethod def test_get_pipeline_templates_fallbacks_to_builtin_for_non_english_empty_result( mocker: MockerFixture, sqlite_session: Session, config_overrides: Callable[..., None] @@ -436,73 +444,6 @@ def test_get_all_published_workflow_applies_limit_and_has_more( # --- sync_draft_workflow --- - @staticmethod - def test_sync_draft_workflow_creates_new_when_none_exists( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - pipeline = _make_pipeline(workflow_id=None) - account = _make_account() - - result = rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": []}, - unique_hash=None, - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - assert result.app_id == "p1" - assert pipeline.workflow_id == result.id - - @staticmethod - def test_sync_draft_workflow_raises_on_hash_mismatch( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - from services.errors.app import WorkflowHashNotEqualError - - existing_wf = _make_workflow(graph={"nodes": [{"id": "old"}]}) - _persist(rag_pipeline_service.session, existing_wf) - - pipeline = _make_pipeline() - account = _make_account() - - with pytest.raises(WorkflowHashNotEqualError): - rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": []}, - unique_hash="hash-different", - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - @staticmethod - def test_sync_draft_workflow_updates_existing( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - existing_wf = _make_workflow(graph={"nodes": [{"id": "old"}]}) - _persist(rag_pipeline_service.session, existing_wf) - - pipeline = _make_pipeline() - account = _make_account() - - result = rag_pipeline_service.service.sync_draft_workflow( - pipeline=pipeline, - graph={"nodes": [{"id": "n1"}]}, - unique_hash=existing_wf.unique_hash, - account=account, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - - assert result is existing_wf - assert result.updated_by == "u1" - assert result.graph_dict == {"nodes": [{"id": "n1"}]} - # --- get_default_block_config --- @staticmethod @@ -537,7 +478,7 @@ def test_update_workflow_updates_allowed_fields( rag_pipeline_service: RagPipelineServiceTestContext, ) -> None: workflow = _make_workflow() - _persist(rag_pipeline_service.session, workflow) + _persist(rag_pipeline_service.session, workflow, _make_account()) workflow_ref = WorkflowRef(tenant_id="t1", owner_id="pipeline-1", workflow_id="wf-1") workflow.app_id = "pipeline-1" rag_pipeline_service.session.commit() @@ -552,7 +493,9 @@ def test_update_workflow_updates_allowed_fields( assert result.marked_name == "v1" assert result.marked_comment == "release" assert not hasattr(result, "disallowed") - assert result.updated_by == "u1" + assert result.updated_by is not None + assert result.updated_by["id"] == "u1" + assert workflow.updated_by == "u1" @staticmethod def test_update_workflow_returns_none_when_not_found( @@ -583,7 +526,9 @@ def test_update_workflow_with_ref_scopes_lookup_to_pipeline( workflow_ref=workflow_ref, ) - assert result is workflow + assert result is not None + assert result.id == workflow.id + assert result.marked_name == workflow.marked_name == "v1" assert other_owner_workflow.marked_name == "" # --- get_rag_pipeline_paginate_workflow_runs --- @@ -630,89 +575,8 @@ def test_get_rag_pipeline_workflow_run_delegates( # --- is_workflow_exist --- - @staticmethod - def test_is_workflow_exist_returns_true_when_draft_exists( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - _persist(rag_pipeline_service.session, _make_workflow()) - - pipeline = _make_pipeline() - assert rag_pipeline_service.service.is_workflow_exist(pipeline) is True - - @staticmethod - def test_is_workflow_exist_returns_false_when_no_draft( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - pipeline = _make_pipeline() - assert rag_pipeline_service.service.is_workflow_exist(pipeline) is False - # --- publish_workflow --- - @staticmethod - def test_publish_workflow_success( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext - ) -> None: - graph = { - "nodes": [ - { - "data": { - "type": "knowledge-index", - "dataset_id": "d1", - "chunk_structure": "paragraph", - "indexing_technique": "high_quality", - "process_rule": {"mode": "automatic"}, - "retrieval_model": {"search_method": "hybrid_search", "top_k": 3}, - } - } - ] - } - draft_workflow = _make_workflow(workflow_id="wf-draft", graph=graph) - pipeline = _make_pipeline(workflow_id="wf-old-published") - dataset = _make_dataset() - _persist(rag_pipeline_service.session, draft_workflow, pipeline, dataset) - mocker.patch( - "services.rag_pipeline.rag_pipeline.KnowledgeConfiguration.model_validate", return_value=mocker.Mock() - ) - mock_dataset_service_class = mocker.patch("services.knowledge.dataset_service.DatasetService") - - result = rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=_make_account() - ) - - assert result.app_id == pipeline.id - assert result.graph_dict == graph - mock_dataset_service_class.update_rag_pipeline_dataset_settings.assert_called_once() - - @staticmethod - def test_publish_workflow_rejects_missing_llm_environment_reference( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - draft_workflow = _make_workflow( - workflow_id="wf-draft", - tenant_id="tenant", - app_id="pipeline", - graph={ - "nodes": [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": {"provider": "provider", "name": "model", "mode": "chat"}, - "model_selector": ["env", "missing_model"], - }, - } - ] - }, - ) - _persist(rag_pipeline_service.session, draft_workflow) - - with pytest.raises(ValueError, match="missing_model.*not found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, - pipeline=_make_pipeline(pipeline_id="pipeline", tenant_id="tenant"), - account=_make_account(account_id="account", tenant_id="tenant"), - ) - # --- run_datasource_workflow_node --- @staticmethod @@ -740,7 +604,7 @@ def test_run_datasource_workflow_node_website_crawl( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) # 2. Mock DatasourceManager and Runtime mock_runtime = mocker.Mock() @@ -762,15 +626,15 @@ def mock_crawl_gen(**kwargs): # 3. Mock DatasourceProviderService mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"api_key": "sk-123"}, ) # 4. Mock Enums to avoid import issues or for consistency - mocker.patch("services.rag_pipeline.rag_pipeline.DatasourceProviderType", DatasourceProviderType) + mocker.patch("services.rag_pipeline.execution_service.DatasourceProviderType", DatasourceProviderType) # 5. Run test - gen = rag_pipeline_service.service.run_datasource_workflow_node( + gen = rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={"url": "https://example.com"}, @@ -823,7 +687,7 @@ def test_run_datasource_node_preview_online_document( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) # 2. Mock Runtime and results mock_runtime = mocker.Mock() @@ -849,13 +713,13 @@ def mock_doc_gen(**kwargs): return_value=mock_runtime, ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"token": "abc"}, ) - mocker.patch("services.rag_pipeline.rag_pipeline.DatasourceProviderType", DatasourceProviderType) + mocker.patch("services.rag_pipeline.execution_service.DatasourceProviderType", DatasourceProviderType) # 3. Run test - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -904,7 +768,7 @@ def mock_getter(): yield event # 2. Run test - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, mock_getter()), start_at=time.perf_counter(), tenant_id="t1", @@ -1087,6 +951,10 @@ def test_retry_error_document_success( # 3. Assertions mock_gen_instance.generate.assert_called_once() + execution_workflow = mock_gen_instance.generate.call_args.kwargs["workflow"] + assert isinstance(execution_workflow, WorkflowSnapshot) + assert execution_workflow == workflow_snapshot(workflow) + assert mock_gen_instance.generate.call_args.kwargs["is_retry"] is True # --- set_datasource_variables --- @@ -1104,10 +972,11 @@ def test_set_datasource_variables_success( ) pipeline.id = "p-1" - draft_wf = mocker.Mock() - draft_wf.id = "wf-1" - draft_wf.get_enclosing_node_type_and_id.return_value = None # Avoid unpacking error - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_wf) + draft_wf = _make_workflow( + workflow_id="wf-1", + graph={"nodes": [{"id": "node-1", "data": {"type": "datasource"}}], "edges": []}, + ) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_wf) execution = WorkflowNodeExecution( id="exec-1", @@ -1129,33 +998,52 @@ def test_set_datasource_variables_success( created_at=datetime(2026, 1, 1), finished_at=None, ) - mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=execution) + run = (mocker.Mock(), (event for event in ())) + mocker.patch("services.rag_pipeline.execution_service.WorkflowEntry.single_step_run", return_value=run) + persist_single_step_run = mocker.patch.object( + rag_pipeline_service.execution, "_persist_single_step_run", return_value=execution + ) # Mock Repository mock_repo_instance = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.SQLAlchemyWorkflowNodeExecutionRepository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=mock_repo_instance, ) - # Repository._to_db_model is also called + # Read the committed execution through the injected repository. mock_db_exec = mocker.Mock() mock_db_exec.node_id = "node-1" mock_db_exec.node_type = "datasource" - mock_repo_instance._to_db_model.return_value = mock_db_exec + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( + get_execution_by_id=mocker.Mock(return_value=mock_db_exec) + ) - # Mock DraftVariableSaver + # Inject through the variable service used by the pipeline. mock_saver_instance = mocker.Mock() - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=mock_saver_instance) + saver_factory = mocker.Mock(return_value=mock_saver_instance) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) # 2. Run test args = {"start_node_id": "node-1"} user = mocker.Mock() user.id = "user-1" - rag_pipeline_service.service.set_datasource_variables(pipeline, args, user) + rag_pipeline_service.execution.set_datasource_variables(pipeline, args, user) # 3. Assertions - mock_repo_instance.save.assert_called_once() - mock_saver_instance.save.assert_called_once() + assert persist_single_step_run.call_args.kwargs["run"] is run + assert persist_single_step_run.call_args.kwargs["repository"] is mock_repo_instance + factory_provider.assert_called_once_with(pipeline.tenant_id, user) + saver_factory.assert_called_once_with( + app_id=pipeline.id, + node_id="node-1", + node_type="datasource", + enclosing_node_id=None, + node_execution_id=execution.id, + ) + mock_saver_instance.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) # --- Utility Methods --- @@ -1201,18 +1089,6 @@ def test_get_default_block_config_success(rag_pipeline_service: RagPipelineServi assert result is not None assert result["type"] == "llm" - @staticmethod - def test_publish_workflow_raises_when_draft_workflow_missing( - rag_pipeline_service: RagPipelineServiceTestContext, - ) -> None: - pipeline = _make_pipeline() - account = _make_account() - - with pytest.raises(ValueError, match="No valid workflow found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=account - ) - @staticmethod def test_get_default_block_config_returns_none_when_mapped_type_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext @@ -1244,10 +1120,10 @@ def test_run_draft_workflow_node_raises_when_workflow_missing( ) -> None: pipeline = _make_pipeline() account = _make_account() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) with pytest.raises(ValueError, match="Workflow not initialized"): - rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {}, account) + rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) @staticmethod def test_run_draft_workflow_node_seeds_llm_environment_variable( @@ -1270,31 +1146,54 @@ def test_run_draft_workflow_node_seeds_llm_environment_variable( "description": "Shared summarization model", } ) - draft_workflow = _make_workflow(workflow_id="wf-1") + draft_workflow = _make_workflow( + workflow_id="wf-1", + graph={"nodes": [{"id": "node-1", "data": {"type": "llm"}}], "edges": []}, + ) draft_workflow.environment_variables = [llm_environment_variable] - mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) - mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=None) - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_workflow) + mocker.patch.object( + rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow + ) - execution = SimpleNamespace(id="exec-1", node_id="node-1", node_type="llm", process_data={}, outputs={}) - handle_node_run_result = mocker.patch.object( - rag_pipeline_service.service, "_handle_node_run_result", return_value=execution + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=draft_workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="llm", + title="LLM", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + process_data={}, + outputs={}, + ) + run = (mocker.Mock(), (event for event in ())) + single_step_run = mocker.patch( + "services.rag_pipeline.execution_service.WorkflowEntry.single_step_run", return_value=run + ) + persist_single_step_run = mocker.patch.object( + rag_pipeline_service.execution, "_persist_single_step_run", return_value=execution ) - single_step_run = mocker.patch("services.rag_pipeline.rag_pipeline.WorkflowEntry.single_step_run") repo = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=repo, ) - rag_pipeline_service.service._node_execution_service_repo = mocker.Mock( + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( get_execution_by_id=mocker.Mock(return_value="db") ) - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=mocker.Mock()) - rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {}, account) + saver = mocker.Mock() + saver_factory = mocker.Mock(return_value=saver) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) + rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) - getter = handle_node_run_result.call_args.kwargs["getter"] - getter() + factory_provider.assert_called_once_with(pipeline.tenant_id, account) + saver.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) + assert persist_single_step_run.call_args.kwargs["run"] is run variable_pool = single_step_run.call_args.kwargs["variable_pool"] llm_model = variable_pool.get(["env", "for_summarize"]) assert llm_model is not None @@ -1310,41 +1209,82 @@ def test_run_draft_workflow_node_saves_execution_and_variables( ) -> None: pipeline = _make_pipeline() account = _make_account() - draft_workflow = _make_workflow(workflow_id="wf-1") - mocker.patch.object(draft_workflow, "get_node_config_by_id", return_value={"id": "node-1"}) - mocker.patch.object(draft_workflow, "get_enclosing_node_type_and_id", return_value=("loop", "enclosing-node")) - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=draft_workflow) + draft_workflow = _make_workflow( + workflow_id="wf-1", + graph={ + "nodes": [ + { + "id": "node-1", + "data": {"type": "llm"}, + "isInLoop": True, + "loop_id": "enclosing-node", + } + ], + "edges": [], + }, + ) + mocker.patch.object( + rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=draft_workflow + ) - execution = SimpleNamespace(id="exec-1", node_id="node-1", node_type="llm", process_data={}, outputs={}) - mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=execution) + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=draft_workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="llm", + title="LLM", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + process_data={}, + outputs={}, + ) + run = (mocker.Mock(), (event for event in ())) + mocker.patch("services.rag_pipeline.execution_service.WorkflowEntry.single_step_run", return_value=run) + persist_single_step_run = mocker.patch.object( + rag_pipeline_service.execution, "_persist_single_step_run", return_value=execution + ) repo = mocker.Mock() - mocker.patch( - "services.rag_pipeline.rag_pipeline.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", + mocker.patch.object( + rag_pipeline_service.execution, + "_writer_factory", return_value=repo, ) - rag_pipeline_service.service._node_execution_service_repo = mocker.Mock( + rag_pipeline_service.execution._node_execution_service_repo = mocker.Mock( get_execution_by_id=mocker.Mock(return_value="db") ) saver = mocker.Mock() - mocker.patch("services.rag_pipeline.rag_pipeline.DraftVariableSaver", return_value=saver) + saver_factory = mocker.Mock(return_value=saver) + factory_provider = mocker.patch.object( + rag_pipeline_service.execution._variables, "saver_factory", return_value=saver_factory + ) - result = rag_pipeline_service.service.run_draft_workflow_node(pipeline, "node-1", {"q": "x"}, account) + result = rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {"q": "x"}, account) assert result == "db" - assert execution.workflow_id == "wf-1" - repo.save.assert_called_once_with(execution) - saver.save.assert_called_once() + assert persist_single_step_run.call_args.kwargs["run"] is run + assert persist_single_step_run.call_args.kwargs["repository"] is repo + assert persist_single_step_run.call_args.kwargs["workflow_id"] == "wf-1" + factory_provider.assert_called_once_with(pipeline.tenant_id, account) + saver_factory.assert_called_once_with( + app_id=pipeline.id, + node_id="node-1", + node_type="llm", + enclosing_node_id="enclosing-node", + node_execution_id=execution.id, + ) + saver.save.assert_called_once_with(process_data=execution.process_data, outputs=execution.outputs) @staticmethod def test_run_datasource_workflow_node_returns_error_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: pipeline = _make_pipeline() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -1382,7 +1322,7 @@ def test_run_datasource_workflow_node_online_document_success( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.runtime = SimpleNamespace(credentials=None) @@ -1392,12 +1332,12 @@ def test_run_datasource_workflow_node_online_document_success( "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={"token": "x"}, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -1439,7 +1379,7 @@ def test_run_datasource_workflow_node_online_drive_success( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.runtime = SimpleNamespace(credentials=None) @@ -1449,12 +1389,12 @@ def test_run_datasource_workflow_node_online_drive_success( "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value={}, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={"bucket": "bucket-1"}, @@ -1476,7 +1416,6 @@ def test_handle_node_run_result_default_value_strategy( ) -> None: from datetime import datetime - from graphon.engine_events import NodeRunFailedEvent from graphon.node_events.base import NodeRunResult node_instance = _make_start_node( @@ -1504,7 +1443,7 @@ def _events(): node_run_result=failed_result, ) - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, _events()), start_at=time.perf_counter(), tenant_id="t1", @@ -1558,20 +1497,20 @@ def test_get_second_step_parameters_handles_string_and_list_variable_references( @staticmethod def test_get_rag_pipeline_workflow_run_node_executions_empty_when_run_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: pipeline = _make_pipeline() mocker.patch.object(rag_pipeline_service.service, "get_rag_pipeline_workflow_run", return_value=None) result = rag_pipeline_service.service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, run_id="run-1", user=_make_account() + pipeline=pipeline, run_id="run-1", user=_make_account(), tool_providers=tool_providers ) assert result == [] @staticmethod def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repository_results( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: pipeline = _make_pipeline() mocker.patch.object( @@ -1587,7 +1526,7 @@ def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repo ) result = rag_pipeline_service.service.get_rag_pipeline_workflow_run_node_executions( - pipeline=pipeline, run_id="run-1", user=_make_account() + pipeline=pipeline, run_id="run-1", user=_make_account(), tool_providers=tool_providers ) assert result == expected_traces @@ -1596,13 +1535,17 @@ def test_get_rag_pipeline_workflow_run_node_executions_assembles_configured_repo app_id=pipeline.id, workflow_run_id="run-1", ) - mock_assemble.assert_called_once_with(expected_executions, node_repo, session=mock.ANY) + mock_assemble.assert_called_once_with( + expected_executions, node_repo, session=mock.ANY, tool_providers=tool_providers + ) @staticmethod def test_get_recommended_plugins_returns_empty_when_no_active_plugins( - rag_pipeline_service: RagPipelineServiceTestContext, + rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: - result = rag_pipeline_service.service.get_recommended_plugins("all", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "all", _make_account(), "t1", tool_providers=tool_providers + ) assert result == { "installed_recommended_plugins": [], @@ -1611,7 +1554,7 @@ def test_get_recommended_plugins_returns_empty_when_no_active_plugins( @staticmethod def test_get_recommended_plugins_returns_installed_and_uninstalled( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: plugin_a = _make_recommended_plugin("plugin-a") plugin_b = _make_recommended_plugin("plugin-b") @@ -1625,7 +1568,9 @@ def test_get_recommended_plugins_returns_installed_and_uninstalled( return_value=[{"plugin_id": "plugin-b", "name": "Plugin B"}], ) - result = rag_pipeline_service.service.get_recommended_plugins("tool", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "tool", _make_account(), "t1", tool_providers=tool_providers + ) assert result["installed_recommended_plugins"] == [{"plugin_id": "plugin-a"}] assert result["uninstalled_recommended_plugins"] == [{"plugin_id": "plugin-b", "name": "Plugin B"}] @@ -1650,10 +1595,10 @@ def test_set_datasource_variables_raises_when_node_id_missing( ) -> None: pipeline = _make_pipeline() workflow = _make_workflow() - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow) with pytest.raises(ValueError, match="Node id is required"): - rag_pipeline_service.service.set_datasource_variables(pipeline, {"start_node_id": ""}, _make_account()) + rag_pipeline_service.execution.set_datasource_variables(pipeline, {"start_node_id": ""}, _make_account()) @staticmethod def test_get_default_block_configs_skips_empty_configs( @@ -1685,10 +1630,10 @@ def test_run_datasource_workflow_node_returns_error_when_node_missing( ) -> None: pipeline = _make_pipeline() workflow = _make_workflow(graph={"nodes": []}) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="missing-node", user_inputs={}, @@ -1724,7 +1669,7 @@ def test_run_datasource_workflow_node_online_document_exception( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() @@ -1742,12 +1687,12 @@ def __next__(self): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -1786,7 +1731,7 @@ def test_run_datasource_node_preview_raises_for_stream_non_string( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() @@ -1803,12 +1748,12 @@ def _bad_stream_generator[**P](*args: P.args, **kwargs: P.kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) with pytest.raises(RuntimeError, match="must be a string"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -1905,7 +1850,7 @@ def _event_generator(): yield object() with pytest.raises(ValueError, match="Node run failed with no run result"): - rag_pipeline_service.service._handle_node_run_result( + rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, _event_generator()), start_at=time.perf_counter(), tenant_id="t1", @@ -1925,7 +1870,7 @@ def test_handle_node_run_result_marks_document_error_for_published_invoke( document, ) - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, iter([event])), start_at=time.perf_counter(), tenant_id="t1", @@ -1945,7 +1890,7 @@ def test_handle_node_run_result_does_not_write_when_pipeline_dataset_is_missing( ) -> None: node_instance, event = _make_failed_published_node_run() - result = rag_pipeline_service.service._handle_node_run_result( + result = rag_pipeline_service.execution._handle_node_run_result( getter=lambda: (node_instance, iter([event])), start_at=time.perf_counter(), tenant_id="t1", @@ -1975,19 +1920,19 @@ def test_run_datasource_node_preview_raises_for_unsupported_provider( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() runtime.datasource_provider_type.return_value = "unsupported" mocker.patch( "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) with pytest.raises(RuntimeError, match="Unsupported datasource provider"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=pipeline, node_id="node-1", user_inputs={}, @@ -2028,8 +1973,10 @@ def test_get_pipeline_raises_when_pipeline_missing( rag_pipeline_service.service.get_pipeline("t1", "d1") @staticmethod - def test_init_uses_default_sessionmaker_when_none(mocker: MockerFixture, sqlite_session: Session) -> None: - mocker.patch("services.rag_pipeline.rag_pipeline.db", SimpleNamespace(engine=sqlite_session.get_bind())) + def test_init_uses_default_sessionmaker_when_none( + mocker: MockerFixture, + sqlite_session: Session, + ) -> None: create_exec_repo = mocker.patch( "services.rag_pipeline.rag_pipeline.DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository" ) @@ -2097,26 +2044,9 @@ def test_get_all_published_workflow_without_filters_has_no_more( named_only=False, ) - assert workflows == [workflow] + assert [record.id for record in workflows] == [workflow.id] assert has_more is False - @staticmethod - def test_publish_workflow_skips_dataset_update_for_non_knowledge_nodes( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext - ) -> None: - draft = _make_workflow(graph={"nodes": [{"data": {"type": "start"}}]}) - _persist(rag_pipeline_service.session, draft) - dataset_service = mocker.patch("services.knowledge.dataset_service.DatasetService") - - result = rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, - pipeline=_make_pipeline(), - account=_make_account(), - ) - - assert result.graph_dict == {"nodes": [{"data": {"type": "start"}}]} - dataset_service.update_rag_pipeline_dataset_settings.assert_not_called() - @staticmethod def test_get_default_block_config_returns_none_when_default_empty( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext @@ -2156,7 +2086,7 @@ def test_run_datasource_workflow_node_handles_variable_parameter_types( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def crawl_gen(**kwargs): @@ -2168,12 +2098,12 @@ def crawl_gen(**kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=_make_pipeline(), node_id="node-1", user_inputs={"k": "mapped"}, @@ -2210,7 +2140,7 @@ def test_run_datasource_workflow_node_online_drive_branch( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def drive_gen(**kwargs): @@ -2222,12 +2152,12 @@ def drive_gen(**kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) events = list( - rag_pipeline_service.service.run_datasource_workflow_node( + rag_pipeline_service.execution.run_datasource_workflow_node( pipeline=_make_pipeline(), node_id="node-1", user_inputs={}, @@ -2264,7 +2194,9 @@ def test_run_datasource_node_preview_not_published_uses_draft( ] } ) - get_draft = mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=workflow) + get_draft = mocker.patch.object( + rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow + ) runtime = mocker.Mock() def doc_gen(**kwargs): @@ -2278,11 +2210,11 @@ def doc_gen(**kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, @@ -2297,24 +2229,6 @@ def doc_gen(**kwargs): assert result == {"x": "v"} get_draft.assert_called_once() - @staticmethod - def test_run_free_workflow_node_delegates_to_handle_result( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext - ) -> None: - expected = SimpleNamespace(id="exec-1") - handle = mocker.patch.object(rag_pipeline_service.service, "_handle_node_run_result", return_value=expected) - - result = rag_pipeline_service.service.run_free_workflow_node( - node_data={"type": "start"}, - tenant_id="t1", - user_id="u1", - node_id="n1", - user_inputs={}, - ) - - assert result is expected - handle.assert_called_once() - @staticmethod @pytest.mark.parametrize(("workflow_tenant_id", "workflow_app_id"), [("t2", "p1"), ("t1", "p2")]) def test_publish_customized_pipeline_template_rejects_unowned_workflow_before_export( @@ -2381,14 +2295,16 @@ def test_publish_customized_pipeline_template_rejects_missing_or_unowned_draft_b @staticmethod def test_get_recommended_plugins_skips_manifest_when_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext + mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext, *, tool_providers ) -> None: plugin = _make_recommended_plugin("plugin-a") _persist(rag_pipeline_service.session, plugin) mocker.patch("services.rag_pipeline.rag_pipeline.BuiltinToolManageService.list_builtin_tools", return_value=[]) mocker.patch("services.rag_pipeline.rag_pipeline.marketplace.batch_fetch_plugin_by_ids", return_value=[]) - result = rag_pipeline_service.service.get_recommended_plugins("all", _make_account(), "t1") + result = rag_pipeline_service.service.get_recommended_plugins( + "all", _make_account(), "t1", tool_providers=tool_providers + ) assert result["installed_recommended_plugins"] == [] assert result["uninstalled_recommended_plugins"] == [] @@ -2458,30 +2374,14 @@ def test_get_datasource_plugins_returns_empty_for_non_datasource_nodes( == [] ) - @staticmethod - def test_publish_workflow_raises_when_knowledge_index_dataset_missing( - mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext - ) -> None: - draft = _make_workflow(graph={"nodes": [{"data": {"type": "knowledge-index"}}]}) - _persist(rag_pipeline_service.session, draft) - mocker.patch( - "services.rag_pipeline.rag_pipeline.KnowledgeConfiguration.model_validate", return_value=mocker.Mock() - ) - pipeline = _make_pipeline() - - with pytest.raises(ValueError, match="Dataset not found"): - rag_pipeline_service.service.publish_workflow( - session=rag_pipeline_service.session, pipeline=pipeline, account=_make_account() - ) - @staticmethod def test_run_datasource_node_preview_raises_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=None) with pytest.raises(RuntimeError, match="Workflow not initialized"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, @@ -2498,11 +2398,13 @@ def test_run_datasource_node_preview_raises_when_node_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: mocker.patch.object( - rag_pipeline_service.service, "get_published_workflow", return_value=_make_workflow(graph={"nodes": []}) + rag_pipeline_service.execution._workflows, + "get_published_workflow", + return_value=_make_workflow(graph={"nodes": []}), ) with pytest.raises(RuntimeError, match="Datasource node data not found"): - rag_pipeline_service.service.run_datasource_node_preview( + rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="missing", user_inputs={}, @@ -2535,7 +2437,7 @@ def test_run_datasource_node_preview_keeps_existing_user_input( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def gen(**kwargs): @@ -2551,11 +2453,11 @@ def gen(**kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={"workspace_id": "existing"}, @@ -2587,7 +2489,7 @@ def test_run_datasource_node_preview_ignores_non_variable_messages( ] } ) - mocker.patch.object(rag_pipeline_service.service, "get_published_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_published_workflow", return_value=workflow) runtime = mocker.Mock() def gen(**kwargs): @@ -2598,11 +2500,11 @@ def gen(**kwargs): "core.datasource.datasource_manager.DatasourceManager.get_datasource_runtime", return_value=runtime ) mocker.patch( - "services.rag_pipeline.rag_pipeline.DatasourceProviderService.get_datasource_credentials", + "services.rag_pipeline.execution_service.DatasourceProviderService.get_datasource_credentials", return_value=None, ) - result = rag_pipeline_service.service.run_datasource_node_preview( + result = rag_pipeline_service.execution.run_datasource_node_preview( pipeline=_make_pipeline(), node_id="n1", user_inputs={}, @@ -2619,10 +2521,10 @@ def gen(**kwargs): def test_set_datasource_variables_raises_when_workflow_missing( mocker: MockerFixture, rag_pipeline_service: RagPipelineServiceTestContext ) -> None: - mocker.patch.object(rag_pipeline_service.service, "get_draft_workflow", return_value=None) + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=None) with pytest.raises(ValueError, match="Workflow not initialized"): - rag_pipeline_service.service.set_datasource_variables( + rag_pipeline_service.execution.set_datasource_variables( _make_pipeline(), {"start_node_id": "n1"}, _make_account(), @@ -2731,3 +2633,136 @@ def test_get_pipeline_by_id_uses_provided_session(sqlite_session: Session) -> No assert result is pipeline assert RagPipelineService.get_pipeline_by_id("p1", "other-tenant", session=sqlite_session) is None + + @staticmethod + def test_single_node_uses_the_injected_variable_store( + rag_pipeline_service: RagPipelineServiceTestContext, + workflow_variables: WorkflowExecutionVariables, + mocker: MockerFixture, + ) -> None: + from repositories.sqlalchemy_api_workflow_node_execution_repository import ( + DifyAPISQLAlchemyWorkflowNodeExecutionRepository, + ) + + pipeline = _make_pipeline() + workflow = _make_workflow(graph={"nodes": [{"id": "node-1", "data": {"type": "code"}}], "edges": []}) + account = _make_account() + assert rag_pipeline_service.execution._variables is workflow_variables + workflow_variables.saver_factory(pipeline.tenant_id, account)( + pipeline.id, "previous", BuiltinNodeTypes.CODE, "previous-execution" + ).save(process_data=None, outputs={"text": "injected input"}) + execution = WorkflowNodeExecution( + id="exec-1", + workflow_id=workflow.id, + created_at=datetime(2026, 1, 1), + node_id="node-1", + node_type="code", + title="Code", + index=1, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + outputs={"text": "injected output"}, + ) + loaded: list[str] = [] + + def execute(*, variable_loader: VariableLoader, **_kwargs: object): + value = variable_loader.load_variables([["previous", "text"]])[0].value + assert isinstance(value, str) + loaded.append(value) + return mocker.Mock(), (event for event in ()) + + def result(getter: Callable[[], object], **_kwargs: object) -> WorkflowNodeExecution: + getter() + return execution + + mocker.patch.object(rag_pipeline_service.execution._workflows, "get_draft_workflow", return_value=workflow) + mocker.patch.object(rag_pipeline_service.execution, "_handle_node_run_result", side_effect=result) + mocker.patch("services.rag_pipeline.execution_service.WorkflowEntry.single_step_run", side_effect=execute) + rag_pipeline_service.execution._node_execution_service_repo = DifyAPISQLAlchemyWorkflowNodeExecutionRepository( + rag_pipeline_service.session_maker + ) + saved = rag_pipeline_service.execution.run_draft_workflow_node(pipeline, "node-1", {}, account) + + assert saved is not None + assert saved.id == execution.id + assert saved.tenant_id == pipeline.tenant_id + assert saved.app_id == pipeline.id + assert loaded == ["injected input"] + loader = workflow_variables.workflow_loader(workflow_snapshot(workflow), account.id) + assert loader.load_variables([["node-1", "text"]])[0].value == "injected output" + + @staticmethod + @pytest.mark.parametrize("entry", ["datasource", "draft"]) + def test_debug_execution_commits_before_variable_save_with_celery_repository( + rag_pipeline_service: RagPipelineServiceTestContext, + workflow_variables: WorkflowExecutionVariables, + monkeypatch: pytest.MonkeyPatch, + entry: str, + ) -> None: + from core.repositories.celery_workflow_node_execution_repository import ( + CeleryWorkflowNodeExecutionRepository, + save_workflow_node_execution_task, + ) + from graphon.engine_events import NodeRunSucceededEvent + from models.workflow import WorkflowNodeExecutionTriggeredFrom + from repositories.sqlalchemy_api_workflow_node_execution_repository import ( + DifyAPISQLAlchemyWorkflowNodeExecutionRepository, + ) + + pipeline = _make_pipeline() + account = _make_account() + workflow = _make_workflow(graph={"nodes": [{"id": "node-1", "data": {"type": "code"}}], "edges": []}) + _persist(rag_pipeline_service.session, workflow) + executions = DifyAPISQLAlchemyWorkflowNodeExecutionRepository(rag_pipeline_service.session_maker) + writer_factory = partial( + CeleryWorkflowNodeExecutionRepository, session_factory=rag_pipeline_service.session_maker + ) + queued = mock.Mock() + monkeypatch.setattr(save_workflow_node_execution_task, "delay", queued) + writer_factory( + tenant_id=pipeline.tenant_id, + user=account, + app_id=pipeline.id, + triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, + ).save( + WorkflowNodeExecution( + id="queued-only", + workflow_id=workflow.id, + node_id="node-1", + node_type="code", + title="Code", + index=1, + created_at=datetime(2026, 1, 1), + ) + ) + queued.assert_called_once() + assert executions.get_execution_by_id("queued-only", tenant_id=pipeline.tenant_id) is None + + node = mock.Mock(workflow_id=workflow.id, node_type="code", title="Code") + result = NodeRunSucceededEvent( + id="event", + start_at=time.time(), + node_id="node-1", + node_type="code", + node_run_result=NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED, outputs={"text": "saved"}), + route_node_id=None, + ) + monkeypatch.setattr( + execution_module.WorkflowEntry, + "single_step_run", + lambda **_kwargs: (node, (event for event in [result])), + ) + service = rag_pipeline_service.execution + service._writer_factory = writer_factory + service._node_execution_service_repo = executions + if entry == "datasource": + saved = service.set_datasource_variables(pipeline, {"start_node_id": "node-1"}, account) + else: + saved = service.run_draft_workflow_node(pipeline, "node-1", {}, account) + assert saved is not None + assert executions.get_execution_by_id(saved.id, tenant_id=pipeline.tenant_id) is not None + assert queued.call_count == 2 + assert queued.call_args_list[-1].kwargs["execution_data"]["status"] == WorkflowNodeExecutionStatus.SUCCEEDED + loaded = workflow_variables.workflow_loader(workflow_snapshot(workflow), account.id).load_variables( + [["node-1", "text"]] + ) + assert loaded[0].value == "saved" diff --git a/api/tests/unit_tests/services/test_agent_service.py b/api/tests/unit_tests/services/test_agent_service.py index 3fc1e493ca556a..161754324202f5 100644 --- a/api/tests/unit_tests/services/test_agent_service.py +++ b/api/tests/unit_tests/services/test_agent_service.py @@ -3,23 +3,66 @@ from __future__ import annotations import json +from dataclasses import dataclass, field from datetime import datetime +from typing import TypedDict, cast import pytest -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker -from models import App +from core.tools.entities.tool_entities import ToolProviderType +from machinery.context import RequestContext from models.enums import CreatorUserRole from models.model import AppMode, AppModelConfig, Conversation, ConversationFromSource, Message, MessageAgentThought -from services import agent_service +from repositories.app.generation_repository import AppGenerationRepository +from services.agent.log_service import AgentLogService from tests.helpers.unit.model_factories import make_account, make_app, make_conversation, make_end_user, make_message _NOW = datetime(2026, 1, 1) +_CONTEXT = RequestContext("request", None, "account-1", "tenant-1") -def _persist_log( - session: Session, monkeypatch: pytest.MonkeyPatch, *, tool_still_configured: bool = False, end_user: bool = False -) -> App: +class _ToolCallResult(TypedDict): + tool_icon: object + + +class _IterationResult(TypedDict): + tool_calls: list[_ToolCallResult] + thought: str | None + + +class _MetaResult(TypedDict): + executor: str + total_tokens: int + iterations: int + + +class _AgentLogResult(TypedDict): + iterations: list[_IterationResult] + meta: _MetaResult + + +class _Files: + def resolve(self, *, tenant_id: str, files: list[dict[str, object]]) -> list[dict[str, object]]: + assert tenant_id == "tenant-1" + assert files == [] + return [] + + +@dataclass +class _ToolIcons: + calls: list[dict[str, str]] = field(default_factory=list) + + def __call__(self, *, tenant_id: str, provider_type: ToolProviderType, provider_id: str) -> str: + self.calls.append({"tenant_id": tenant_id, "provider_type": provider_type.value, "provider_id": provider_id}) + return "icon-url" + + +def _service(sessions: sessionmaker[Session], icons: _ToolIcons | None = None) -> AgentLogService: + return AgentLogService(records=AppGenerationRepository(sessions), icons=icons or _ToolIcons(), files=_Files()) + + +def _persist_log(session: Session, *, tool_still_configured: bool = False, end_user: bool = False) -> None: account = make_account(name="Executor", timezone="UTC") config = AppModelConfig( app_id="app-1", @@ -74,25 +117,29 @@ def _persist_log( session.add(make_end_user(app_id=app.id, name="Visitor")) session.commit() session.expire_all() - monkeypatch.setattr(agent_service, "current_user", account) - return app + + +def _get_log( + service: AgentLogService, *, conversation_id: str = "conversation-1", message_id: str = "message-1" +) -> _AgentLogResult: + return cast( + _AgentLogResult, + service.get(_CONTEXT, app_id="app-1", conversation_id=conversation_id, message_id=message_id), + ) @pytest.mark.parametrize("tool_still_configured", [False, True]) @pytest.mark.parametrize("end_user", [False, True]) def test_legacy_tool_log_without_provider_metadata_uses_configured_tool_or_blank_icon( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, tool_still_configured: bool, end_user: bool + sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], + tool_still_configured: bool, + end_user: bool, ) -> None: - app = _persist_log(sqlite_session, monkeypatch, tool_still_configured=tool_still_configured, end_user=end_user) - calls: list[dict[str, str]] = [] - - def get_tool_icon(*, tenant_id: str, provider_type: str, provider_id: str) -> str: - calls.append({"tenant_id": tenant_id, "provider_type": provider_type, "provider_id": provider_id}) - return "icon-url" - - monkeypatch.setattr(agent_service.ToolManager, "get_tool_icon", get_tool_icon) + _persist_log(sqlite_session, tool_still_configured=tool_still_configured, end_user=end_user) + icons = _ToolIcons() - result = agent_service.AgentService.get_agent_logs(app, "conversation-1", "message-1", sqlite_session) + result = _get_log(_service(sqlite_session_factory, icons)) assert result["iterations"][0]["tool_calls"][0]["tool_icon"] == ("icon-url" if tool_still_configured else "") assert result["meta"]["executor"] == ("Visitor" if end_user else "Executor") @@ -103,28 +150,28 @@ def get_tool_icon(*, tenant_id: str, provider_type: str, provider_id: str) -> st if tool_still_configured else [] ) - assert calls == expected_calls + assert icons.calls == expected_calls @pytest.mark.parametrize("missing", ["conversation", "message"]) def test_agent_logs_reject_missing_records( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, missing: str + sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], missing: str ) -> None: - app = _persist_log(sqlite_session, monkeypatch) + _persist_log(sqlite_session) conversation_id = "missing" if missing == "conversation" else "conversation-1" message_id = "missing" if missing == "message" else "message-1" with pytest.raises( ValueError, match="Conversation not found" if missing == "conversation" else "Message not found" ): - agent_service.AgentService.get_agent_logs(app, conversation_id, message_id, sqlite_session) + _get_log(_service(sqlite_session_factory), conversation_id=conversation_id, message_id=message_id) @pytest.mark.parametrize("foreign", ["conversation", "message"]) def test_agent_logs_reject_records_outside_the_requested_owner_chain( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, foreign: str + sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], foreign: str ) -> None: - app = _persist_log(sqlite_session, monkeypatch) + _persist_log(sqlite_session) other_app = make_app(app_id="other-app") other_conversation = make_conversation( conversation_id="other-conversation", app_id=other_app.id, from_source=ConversationFromSource.CONSOLE @@ -145,13 +192,13 @@ def test_agent_logs_reject_records_outside_the_requested_owner_chain( with pytest.raises( ValueError, match="Conversation not found" if foreign == "conversation" else "Message not found" ): - agent_service.AgentService.get_agent_logs(app, "conversation-1", "message-1", sqlite_session) + _get_log(_service(sqlite_session_factory)) def test_agent_logs_read_only_the_messages_thoughts_in_position_order( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] ) -> None: - app = _persist_log(sqlite_session, monkeypatch) + _persist_log(sqlite_session) other_message = make_message( message_id="other-message", query="Other", @@ -177,7 +224,7 @@ def test_agent_logs_read_only_the_messages_thoughts_in_position_order( sqlite_session.commit() sqlite_session.expire_all() - result = agent_service.AgentService.get_agent_logs(app, "conversation-1", "message-1", sqlite_session) + result = _get_log(_service(sqlite_session_factory)) assert [iteration["thought"] for iteration in result["iterations"]] == ["historical thought", "middle", "last"] assert result["meta"]["iterations"] == 3 diff --git a/api/tests/unit_tests/services/test_agent_tool_inner_service.py b/api/tests/unit_tests/services/test_agent_tool_inner_service.py deleted file mode 100644 index 9d3a9967e7394b..00000000000000 --- a/api/tests/unit_tests/services/test_agent_tool_inner_service.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Unit tests for the Agent tool inner invoke service.""" - -from collections.abc import Generator -from dataclasses import dataclass, field -from unittest.mock import patch - -import pytest -from sqlalchemy.orm import Session - -from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom -from core.tools.__base.tool_runtime import ToolRuntime -from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage, ToolProviderType -from core.tools.errors import ( - ToolInvokeError, - ToolParameterValidationError, - ToolProviderCredentialValidationError, - ToolProviderNotFoundError, -) -from models.model import App, AppMode -from services.agent_tool_inner_service import AgentToolInnerService -from services.entities.agent_tool_inner import AgentToolInvokeRequest -from services.errors.agent_tool_inner import AgentToolInnerServiceError - - -def _request() -> AgentToolInvokeRequest: - return AgentToolInvokeRequest.model_validate( - { - "caller": { - "tenant_id": "tenant-1", - "user_id": "user-1", - "user_from": "account", - "app_id": "app-1", - "invoke_from": "service-api", - "conversation_id": "conversation-1", - "workflow_id": "workflow-1", - "workflow_run_id": "workflow-run-1", - "node_id": "node-1", - "node_execution_id": "node-exec-1", - "agent_id": "agent-1", - "agent_config_version_id": "snapshot-1", - "trace_session_id": "trace-session-1", - }, - "tool": { - "provider_type": "plugin", - "provider_id": "langgenius/search/search", - "tool_name": "search", - "credential_id": "credential-1", - "tool_parameters": {"query": "dify"}, - "runtime_parameters": {"region": "us"}, - }, - } - ) - - -@dataclass -class _FakeTool: - """Stands in for the resolved agent tool; only its runtime is mutated by the service.""" - - runtime: ToolRuntime = field(default_factory=lambda: ToolRuntime(tenant_id="tenant-1")) - - -def _persist_app(session: Session, *, tenant_id: str = "tenant-1") -> App: - app = App( - id="app-1", - tenant_id=tenant_id, - name="Agent tool gateway test app", - description="", - mode=AppMode.AGENT_CHAT, - enable_site=False, - enable_api=False, - max_active_requests=None, - ) - session.add(app) - session.commit() - return app - - -def _messages() -> Generator[ToolInvokeMessage]: - yield ToolInvokeMessage( - type=ToolInvokeMessage.MessageType.TEXT, - message=ToolInvokeMessage.TextMessage(text="ok"), - ) - - -@pytest.mark.parametrize( - ("workflow_id", "expected_tool_invoke_from"), - [ - ("workflow-1", ToolInvokeFrom.WORKFLOW), - (None, ToolInvokeFrom.AGENT), - ], -) -def test_invoke_uses_agent_tool_runtime_and_returns_observation( - sqlite_session: Session, - workflow_id: str | None, - expected_tool_invoke_from: ToolInvokeFrom, -) -> None: - fake_tool = _FakeTool() - _persist_app(sqlite_session) - - with ( - patch( - "services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", - return_value=fake_tool, - ) as mock_get_runtime, - patch("services.agent_tool_inner_service.ToolEngine.generic_invoke", return_value=_messages()) as mock_invoke, - patch( - "services.agent_tool_inner_service.ToolFileMessageTransformer.transform_tool_invoke_messages", - side_effect=lambda messages, **_kwargs: messages, - ), - ): - request = _request() - request.caller.workflow_id = workflow_id - response = AgentToolInnerService().invoke(request, session=sqlite_session) - - assert response.observation == "ok" - assert response.metadata == { - "provider_type": "plugin", - "provider_id": "langgenius/search/search", - "tool_name": "search", - } - agent_tool = mock_get_runtime.call_args.kwargs["agent_tool"] - assert agent_tool.provider_type is ToolProviderType.PLUGIN - assert agent_tool.tool_parameters == {"region": "us"} - assert fake_tool.runtime.dify_run_context is not None - assert fake_tool.runtime.dify_run_context.tenant_id == "tenant-1" - assert fake_tool.runtime.dify_run_context.app_id == "app-1" - assert fake_tool.runtime.dify_run_context.user_id == "user-1" - assert fake_tool.runtime.dify_run_context.user_from is UserFrom.ACCOUNT - assert fake_tool.runtime.dify_run_context.invoke_from is InvokeFrom.SERVICE_API - assert fake_tool.runtime.dify_run_context.trace_session_id == "trace-session-1" - assert fake_tool.runtime.tool_invoke_from is expected_tool_invoke_from - mock_invoke.assert_called_once() - - -def test_invoke_raises_app_not_found_when_session_has_no_app(sqlite_session: Session) -> None: - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_not_found" - assert exc_info.value.status_code == 404 - assert exc_info.value.description == "App not found." - - -def test_invoke_raises_app_tenant_mismatch_when_app_belongs_to_other_tenant(sqlite_session: Session) -> None: - _persist_app(sqlite_session, tenant_id="tenant-2") - - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_tenant_mismatch" - assert exc_info.value.status_code == 403 - assert exc_info.value.description == "App does not belong to the caller tenant." - - -def test_invoke_maps_tool_runtime_app_not_found_value_error_to_specific_error_code(sqlite_session: Session) -> None: - fake_tool = _FakeTool() - _persist_app(sqlite_session) - - with ( - patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", return_value=fake_tool), - patch("services.agent_tool_inner_service.ToolEngine.generic_invoke", side_effect=ValueError("app not found")), - ): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "app_not_found" - assert exc_info.value.status_code == 404 - assert exc_info.value.description == "App not found." - - -def test_invoke_maps_tool_invoke_error_without_private_tool_engine_helper(sqlite_session: Session) -> None: - fake_tool = _FakeTool() - _persist_app(sqlite_session) - - with ( - patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", return_value=fake_tool), - patch( - "services.agent_tool_inner_service.ToolEngine.generic_invoke", - side_effect=ToolInvokeError("workflow crashed"), - ), - ): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == "agent_tool_invoke_failed" - - -@pytest.mark.parametrize( - ("error", "expected_code"), - [ - (ToolProviderNotFoundError("provider gone"), "agent_tool_declaration_not_found"), - (ToolProviderCredentialValidationError("credential invalid"), "agent_tool_credential_invalid"), - (ToolParameterValidationError("query is required"), "tool_parameters_invalid"), - ], -) -def test_invoke_maps_runtime_lookup_errors_to_service_error_codes( - sqlite_session: Session, error: Exception, expected_code: str -) -> None: - _persist_app(sqlite_session) - - with patch("services.agent_tool_inner_service.ToolManager.get_agent_tool_runtime", side_effect=error): - with pytest.raises(AgentToolInnerServiceError) as exc_info: - AgentToolInnerService().invoke(_request(), session=sqlite_session) - - assert exc_info.value.error_code == expected_code diff --git a/api/tests/unit_tests/services/test_app_definition_query_service.py b/api/tests/unit_tests/services/test_app_definition_query_service.py index 1bcf4093f49d6d..27210abc631361 100644 --- a/api/tests/unit_tests/services/test_app_definition_query_service.py +++ b/api/tests/unit_tests/services/test_app_definition_query_service.py @@ -5,10 +5,10 @@ from sqlalchemy.orm import Session, sessionmaker import services.app_definition_query_service as module -from core.app.apps.agent_app.errors import AgentAppGeneratorError, AgentAppNotPublishedError from models.account import Tenant from models.model import App, AppMode from repositories.app_definition_query_repository import AppDefinitionQueryRepository +from services.app.generation.errors import AgentAppGeneratorError, AgentAppNotPublishedError from services.app_definition_query_service import ( AppDefinitionNotPublishedError, AppDefinitionQuery, diff --git a/api/tests/unit_tests/services/test_app_dsl_service.py b/api/tests/unit_tests/services/test_app_dsl_service.py index 3e1263a6492e00..1edef3a132a46f 100644 --- a/api/tests/unit_tests/services/test_app_dsl_service.py +++ b/api/tests/unit_tests/services/test_app_dsl_service.py @@ -1,9 +1,8 @@ from collections.abc import Callable from dataclasses import dataclass, field from io import BytesIO -from types import SimpleNamespace from typing import cast -from unittest.mock import Mock +from unittest.mock import Mock, create_autospec from uuid import uuid4 import pytest @@ -17,19 +16,23 @@ from core.rbac import RBACPermission, RBACResourceScope from core.workflow.llm_environment_variable import LLMEnvironmentVariable from enums import DeploymentEdition +from extensions.application_services.workflow import build_app_dsl_service from models import Account, App, AppMode from models.enums import CustomizeTokenStrategy from models.model import AppModelConfig, AppModelConfigDict, IconType, Site from models.workflow import Workflow from services.agent.dsl_entities import AgentPackage from services.agent.roster_package_entities import PackageIcon +from services.agent.workflow_contracts import WorkflowAgentBindingStore from services.app_dsl_service import AppDslService, PendingData from services.app_package_service import PreparedAppPackage -from services.entities.dsl_entities import ImportStatus +from services.entities.dsl_entities import AppDslOverwriteStore, AppDslOverwriteTarget, ImportStatus from services.entities.feature_entities import FeatureModel, LimitationModel from services.entities.site_dsl import SiteDsl from services.errors.app import DraftChangedError, WorkflowNotFoundError from services.errors.base import NoPermissionError +from services.workflow.contracts import DraftSyncCommand +from services.workflow.draft_service import WorkflowDraftService from services.workflow.graph_diff import draft_token from tests.helpers.unit.config_override import apply_config_overrides from tests.helpers.unit.model_factories import make_account, make_app, make_tenant, make_workflow @@ -120,7 +123,11 @@ def unexpected_side_effect(*_args: object, **_kwargs: object) -> None: monkeypatch.setattr("services.app_dsl_service.FeatureService.get_features", features) monkeypatch.setattr("services.app_dsl_service.redis_client.setex", unexpected_side_effect) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) monkeypatch.setattr(service, "_create_or_update_app", unexpected_side_effect) package = PreparedAppPackage( archive=BytesIO(), @@ -176,11 +183,11 @@ def features(tenant_id: str, *, exclude_vector_space: bool) -> FeatureModel: return FeatureModel(apps=LimitationModel(size=size, limit=limit)) monkeypatch.setattr("services.app_dsl_service.FeatureService.get_features", features) - monkeypatch.setattr( - "services.app_dsl_service.WorkflowDraftVariableService.delete_app_workflow_variables", - lambda *_args, **_kwargs: None, + service = AppDslService( + sqlite_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) - service = AppDslService(sqlite_session) app = _app(mode=AppMode.WORKFLOW) imported: list[App] = [] @@ -229,7 +236,11 @@ def test_confirm_import_rechecks_cloud_app_quota_and_preserves_pending_on_reject ) monkeypatch.setattr("services.app_dsl_service.redis_client.get", pending_imports.get) monkeypatch.setattr("services.app_dsl_service.redis_client.delete", pending_imports.pop) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) imported: list[App] = [] def create_or_update(**_kwargs: object) -> App: @@ -388,7 +399,11 @@ def test_import_app_rejects_oversized_yaml_content_before_parsing( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: monkeypatch.setattr("services.app_dsl_service.DSL_MAX_SIZE", 3) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) account = _account() result = service.import_app(account=account, import_mode="yaml-content", yaml_content="你你") @@ -406,7 +421,11 @@ def test_import_app_rejects_oversized_yaml_url_bytes_before_decode( response.raise_for_status.return_value = None response.content = b"\xff\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) result = service.import_app( account=_account(), @@ -426,7 +445,11 @@ def test_import_app_returns_decode_error_for_invalid_yaml_url_bytes( response.raise_for_status.return_value = None response.content = b"\xff" monkeypatch.setattr("services.app_dsl_service.remote_fetcher.make_request", Mock(return_value=response)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) result = service.import_app( account=_account(), @@ -456,7 +479,7 @@ def deny_before_transaction(*_args: object, **_kwargs: object) -> bool: monkeypatch.setattr("services.app_dsl_service.redis_client.setex", setex) with pytest.raises(NoPermissionError, match="permission to overwrite"): - AppDslService(sqlite_session).import_app( + build_app_dsl_service(sqlite_session).import_app( account=account, import_mode="yaml-content", yaml_content=_PENDING_WORKFLOW_DSL, @@ -481,7 +504,7 @@ def test_confirm_import_rechecks_overwrite_rbac_before_database_access( redis_delete = Mock() monkeypatch.setattr("services.app_dsl_service.redis_client.delete", redis_delete) create_or_update = Mock() - service = AppDslService(sqlite_session) + service = build_app_dsl_service(sqlite_session) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) def deny_before_transaction(*_args: object, **_kwargs: object) -> bool: @@ -517,7 +540,7 @@ def test_confirm_import_does_not_create_when_overwrite_target_disappeared( monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=True)) redis_delete = Mock() monkeypatch.setattr("services.app_dsl_service.redis_client.delete", redis_delete) - service = AppDslService(sqlite_session) + service = build_app_dsl_service(sqlite_session) create_or_update = Mock() monkeypatch.setattr(service, "_create_or_update_app", create_or_update) @@ -538,7 +561,11 @@ def test_pending_import_is_scoped_to_its_owner(monkeypatch: pytest.MonkeyPatch, "services.app_dsl_service.redis_client.setex", lambda key, _expiry, value: pending_imports.__setitem__(key, value), ) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) creator = _account() pending = service.import_app( @@ -588,7 +615,11 @@ def test_pending_import_requires_current_tenant( account = _account() account._current_tenant = None - result = AppDslService(session=unbound_session).import_app( + result = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ).import_app( account=account, import_mode="yaml-content", yaml_content="version: 99.0.0\nkind: app\napp: {name: Test, mode: workflow}\n", @@ -660,7 +691,7 @@ def test_create_or_update_app_loads_existing_model_config_with_service_session( app = _app(app_model_config_id=app_model_config_id) with sqlite_session_factory() as service_session: - result = AppDslService(session=service_session)._create_or_update_app( + result = build_app_dsl_service(session=service_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {"model": {}}}, account=_account(), @@ -674,7 +705,7 @@ def test_create_or_update_app_loads_existing_model_config_with_service_session( def test_create_or_update_app_silently_discards_invalid_image_icon(sqlite_session: Session) -> None: app = _app(tenant_id=_TENANT_ID) - service = AppDslService(session=sqlite_session) + service = build_app_dsl_service(session=sqlite_session) result = service._create_or_update_app( app=app, @@ -711,7 +742,7 @@ def test_create_or_update_app_applies_site_settings_without_changing_access( sqlite_session.add_all([app, site]) sqlite_session.flush() - service = AppDslService(session=sqlite_session) + service = build_app_dsl_service(session=sqlite_session) service._create_or_update_app( app=app, data={ @@ -745,7 +776,11 @@ def test_create_or_update_app_rejects_null_required_site_setting_before_mutation app = _app() with pytest.raises(ValueError, match="Required Site settings cannot be null"): - AppDslService(unbound_session)._create_or_update_app( + AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + )._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT.value}, "site": {"title": None}}, account=_account(), @@ -757,12 +792,15 @@ def test_create_or_update_app_rejects_null_required_site_setting_before_mutation def test_import_app_resolves_site_entitlement_before_database_writes( monkeypatch: pytest.MonkeyPatch, unbound_session: Session ) -> None: - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) create_or_update = Mock(return_value=_app()) entitlement = Mock(return_value=False) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) monkeypatch.setattr("services.app_dsl_service.FeatureService.can_import_premium_site_settings", entitlement) - monkeypatch.setattr("services.app_dsl_service.WorkflowDraftVariableService", Mock()) result = service.import_app( account=_account(), @@ -811,7 +849,11 @@ def test_confirm_import_rechecks_site_entitlement(monkeypatch: pytest.MonkeyPatc monkeypatch.setattr("services.app_dsl_service.redis_client.delete", Mock()) entitlement = Mock(return_value=False) monkeypatch.setattr("services.app_dsl_service.FeatureService.can_import_premium_site_settings", entitlement) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) create_or_update = Mock(return_value=_app()) monkeypatch.setattr(service, "_create_or_update_app", create_or_update) @@ -892,7 +934,7 @@ def record_signal(*_args: object, **_kwargs: object) -> None: app = _app() try: - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {"model": {}}}, account=_account(account_id="22222222-2222-2222-2222-222222222222"), @@ -923,7 +965,7 @@ def test_chat_dsl_import_rejects_invalid_feature_fields( app = _app() with pytest.raises(ValueError, match=field): - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": {feature: {"enabled": True, field: value}}}, account=_account(), @@ -940,7 +982,7 @@ def test_chat_dsl_import_preserves_valid_feature_fields(sqlite_session: Session) "text_to_speech": {"enabled": True, "voice": "alloy", "language": "en", "autoPlay": "disabled"}, } - AppDslService(session=sqlite_session)._create_or_update_app( + build_app_dsl_service(session=sqlite_session)._create_or_update_app( app=app, data={"app": {"mode": AppMode.CHAT}, "model_config": model_config}, account=_account(), @@ -953,23 +995,14 @@ def test_chat_dsl_import_preserves_valid_feature_fields(sqlite_session: Session) assert persisted.text_to_speech_dict == model_config["text_to_speech"] -def test_create_or_update_app_removes_imported_workflow_viewport(monkeypatch: pytest.MonkeyPatch) -> None: - session = cast(Session, SimpleNamespace(add=Mock(), flush=Mock(), get=Mock())) - service = AppDslService(session=session) - app = SimpleNamespace( - id="app-1", - tenant_id="tenant-1", - name="Workflow", - description="", - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - ) - workflow_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=None), - sync_draft_workflow=Mock(return_value=SimpleNamespace(id="workflow-1")), +def test_create_or_update_app_removes_imported_workflow_viewport(unbound_session: Session) -> None: + drafts = create_autospec(WorkflowDraftService, instance=True, spec_set=True) + service = AppDslService( + session=unbound_session, + drafts=drafts, + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), ) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", Mock(return_value=workflow_service)) + app = make_app(mode=AppMode.WORKFLOW) imported_graph: dict[str, object] = { "nodes": [], "edges": [], @@ -977,74 +1010,21 @@ def test_create_or_update_app_removes_imported_workflow_viewport(monkeypatch: py } service._create_or_update_app( - app=cast(App, app), + app=app, data={ "app": {"mode": AppMode.WORKFLOW.value}, "workflow": {"graph": imported_graph}, }, - account=Mock(id="account-1"), + account=make_account(), ) - assert workflow_service.sync_draft_workflow.call_args.kwargs["graph"] == { + assert drafts.sync.call_args.args[2].graph == { "nodes": [], "edges": [], } assert imported_graph["viewport"] == {"x": 100, "y": 200, "zoom": 1.5} -def test_create_or_update_app_forwards_imported_agent_purge_ids(monkeypatch: pytest.MonkeyPatch) -> None: - session = cast(Session, SimpleNamespace(add=Mock(), flush=Mock(), commit=Mock(), get=Mock())) - service = AppDslService(session=session) - app = SimpleNamespace( - id="app-1", - tenant_id="tenant-1", - name="Workflow", - description="", - icon_type=IconType.EMOJI, - icon="robot", - icon_background="#FFFFFF", - ) - workflow = SimpleNamespace(id="workflow-1") - workflow_service = SimpleNamespace( - get_draft_workflow=Mock(return_value=None), - sync_draft_workflow=Mock(return_value=workflow), - ) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", Mock(return_value=workflow_service)) - monkeypatch.setattr( - "services.app_dsl_service.AgentDslService.graph_without_package_bindings", - Mock(return_value={"nodes": [], "edges": []}), - ) - monkeypatch.setattr( - "services.app_dsl_service.AgentDslService.import_workflow_packages", - Mock(return_value=(workflow, [], {"retired-agent"})), - ) - monkeypatch.setattr( - "services.app_dsl_service.WorkflowAgentPublishService.validate_agent_nodes_for_draft_sync", - Mock(), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.app_dsl_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - - service._create_or_update_app( - app=cast(App, app), - data={ - "app": {"mode": AppMode.WORKFLOW.value}, - "workflow": {"graph": {"nodes": [], "edges": []}}, - "agent_packages": {"package-1": {}}, - }, - account=Mock(id="account-1"), - ) - - retire_unowned.assert_called_once_with( - tenant_id="tenant-1", - agent_ids={"retired-agent"}, - account_id="account-1", - ) - - def test_export_dsl_loads_model_config_and_annotation_reply_with_request_session( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], @@ -1088,7 +1068,11 @@ def test_agent_import_of_new_agent_checks_function_scope( monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", check) account = _account() - AppDslService(session=unbound_session)._ensure_agent_import_permission(account, app=None) + AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + )._ensure_agent_import_permission(account, app=None) check.assert_called_once_with( account.current_tenant_id, @@ -1106,7 +1090,11 @@ def test_create_or_update_app_gates_agent_mode_before_creation( ) -> None: config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) with pytest.raises(NoPermissionError): service._create_or_update_app( @@ -1125,7 +1113,11 @@ def test_import_app_reraises_permission_denial_instead_of_failed_result( ) -> None: config_overrides(RBAC_ENABLED=True) monkeypatch.setattr("services.app_dsl_service.RBACService.CheckAccess.check", Mock(return_value=False)) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) with pytest.raises(NoPermissionError): service.import_app( @@ -1219,15 +1211,16 @@ def append_model(export_data: dict[str, object], *_args: object, **_kwargs: obje (AppMode.ADVANCED_CHAT, "trigger-plugin"), ], ) -def test_overwrite_rejects_incompatible_nodes_before_mutation( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, mode: AppMode, node_type: str -) -> None: - apply_config_overrides(monkeypatch, RBAC_ENABLED=False) - target = _app(tenant_id="tenant-1", mode=mode) - target.name = "Original" - sqlite_session.add(target) - sqlite_session.commit() - service = AppDslService(sqlite_session) +def test_overwrite_rejects_incompatible_nodes_before_mutation(mode: AppMode, node_type: str) -> None: + session = Mock() + target = App(id="target", tenant_id="tenant", mode=mode, name="Original") + overwrites = create_autospec(AppDslOverwriteStore, instance=True, spec_set=True) + overwrites.snapshot.return_value = AppDslOverwriteTarget(mode, None) + service = AppDslService( + session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=overwrites, + ) result = service.import_app( account=_account(), import_mode="yaml-content", @@ -1244,10 +1237,9 @@ def test_overwrite_rejects_incompatible_nodes_before_mutation( assert result.status == ImportStatus.FAILED assert result.error is not None assert "incompatible" in result.error - sqlite_session.refresh(target) assert target.name == "Original" - assert not sqlite_session.new - assert not sqlite_session.deleted + session.add.assert_not_called() + overwrites.load.assert_not_called() @pytest.mark.parametrize( @@ -1262,9 +1254,11 @@ def test_overwrite_rejects_incompatible_nodes_before_mutation( def test_missing_app_section_names_the_keys_that_were_present( unbound_session: Session, content: dict[str, object], found: str ) -> None: - result = AppDslService(session=unbound_session).import_app( - account=_account(), import_mode="yaml-content", yaml_content=yaml.safe_dump(content, sort_keys=False) - ) + result = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ).import_app(account=_account(), import_mode="yaml-content", yaml_content=yaml.safe_dump(content, sort_keys=False)) assert result.status == ImportStatus.FAILED assert result.error is not None assert result.error.startswith("Missing app data in YAML content.") @@ -1276,22 +1270,18 @@ def test_missing_app_section_names_the_keys_that_were_present( class _DraftWorkflowService: draft: Workflow | None lookups: list[str] = field(default_factory=list) - synced_hashes: list[str | None] = field(default_factory=list) + syncs: list[DraftSyncCommand] = field(default_factory=list) def _draft_of(self, app_model: App) -> Workflow | None: return self.draft if self.draft is not None and self.draft.app_id == app_model.id else None - def get_draft_workflow(self, app_model: App, **_: object) -> Workflow | None: - self.lookups.append("read") + def get_draft_workflow(self, app_model: App, *, for_update: bool = False, **_: object) -> Workflow | None: + self.lookups.append("locked" if for_update else "read") return self._draft_of(app_model) - def get_draft_workflow_for_update(self, app_model: App, **_: object) -> Workflow | None: - self.lookups.append("locked") - return self._draft_of(app_model) - - def sync_draft_workflow(self, *, graph: dict[str, object], unique_hash: str | None, **_: object) -> Workflow: - self.synced_hashes.append(unique_hash) - return _workflow(graph=graph) + def sync(self, _context: object, _owner: object, command: DraftSyncCommand, **_: object) -> Workflow: + self.syncs.append(command) + return _workflow(graph=command.graph) def _workflow_draft(app: App) -> Workflow: @@ -1301,9 +1291,16 @@ def _workflow_draft(app: App) -> Workflow: def _guarded_overwrite( monkeypatch: pytest.MonkeyPatch, session: Session, *, draft: Workflow | None ) -> tuple[AppDslService, _DraftWorkflowService, Callable[[str], App]]: - service = AppDslService(session=session) workflow_service = _DraftWorkflowService(draft=draft) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", lambda: workflow_service) + service = AppDslService( + session=session, + drafts=cast(WorkflowDraftService[WorkflowAgentBindingStore], workflow_service), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) + monkeypatch.setattr( + "services.app_dsl_service.WorkflowDefinitionStore.get_draft_workflow", + workflow_service.get_draft_workflow, + ) app = _app(mode=AppMode.WORKFLOW) def overwrite(draft_hash: str) -> App: @@ -1326,7 +1323,8 @@ def test_guarded_overwrite_saves_against_the_locked_draft_it_matched( overwrite(draft_token(draft)) assert workflow_service.lookups == ["locked"] - assert workflow_service.synced_hashes == [draft.unique_hash] + assert len(workflow_service.syncs) == 1 + assert workflow_service.syncs[0].graph == {"nodes": [], "edges": []} @pytest.mark.parametrize("draft_exists", [True, False], ids=["draft-changed", "draft-gone"]) @@ -1339,7 +1337,7 @@ def test_guarded_overwrite_refuses_a_draft_that_no_longer_matches( with pytest.raises(DraftChangedError): overwrite("exported") - assert workflow_service.synced_hashes == [] + assert workflow_service.syncs == [] def test_refused_overwrite_leaves_the_app_and_cached_dependencies_alone( @@ -1365,7 +1363,7 @@ def test_refused_overwrite_leaves_the_app_and_cached_dependencies_alone( assert cached == [] assert app.name == "Existing app" - assert workflow_service.synced_hashes == [] + assert workflow_service.syncs == [] def _refuse_changed_draft(forwarded: list[str | None]) -> Callable[..., App]: @@ -1384,7 +1382,11 @@ def test_import_app_reraises_a_changed_draft_and_keeps_the_hash_for_the_confirm( "services.app_dsl_service.redis_client.setex", lambda key, _expiry, value: pending_imports.__setitem__(key, value), ) - service = AppDslService(session=unbound_session) + service = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) forwarded: list[str | None] = [] monkeypatch.setattr(service, "_create_or_update_app", _refuse_changed_draft(forwarded)) @@ -1425,7 +1427,11 @@ def test_confirm_import_checks_the_pending_draft_hash( monkeypatch.setattr("services.app_dsl_service.redis_client.get", lambda _key: pending.model_dump_json()) deleted: list[str] = [] monkeypatch.setattr("services.app_dsl_service.redis_client.delete", deleted.append) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) forwarded: list[str | None] = [] monkeypatch.setattr(service, "_create_or_update_app", _refuse_changed_draft(forwarded)) @@ -1456,8 +1462,7 @@ def test_import_creation_marker_tracks_persisted_app( "workflow": {"graph": {"nodes": [], "edges": []}, "features": {}}, } ) - monkeypatch.setattr("services.app_dsl_service.WorkflowService", Mock()) - service = AppDslService(sqlite_session) + service = build_app_dsl_service(sqlite_session) if confirmed: pending = PendingData( tenant_id=_TENANT_ID, @@ -1505,7 +1510,11 @@ def test_failed_import_does_not_report_creation( sqlite_session: Session, monkeypatch: pytest.MonkeyPatch, confirmed: bool ) -> None: data = "version: " + CURRENT_APP_DSL_VERSION + "\nkind: app\napp: {name: Broken, mode: chat}\n" - service = AppDslService(sqlite_session) + service = AppDslService( + sqlite_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) if confirmed: pending = PendingData( tenant_id="tenant-1", account_id="account-1", import_mode="yaml-content", yaml_content=data @@ -1523,9 +1532,11 @@ def test_pending_import_does_not_report_creation_or_cache_marker( ) -> None: cache = Mock() monkeypatch.setattr("services.app_dsl_service.redis_client.setex", cache) - result = AppDslService(unbound_session).import_app( - account=_account(), import_mode="yaml-content", yaml_content=_PENDING_WORKFLOW_DSL - ) + result = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ).import_app(account=_account(), import_mode="yaml-content", yaml_content=_PENDING_WORKFLOW_DSL) assert result.status == ImportStatus.PENDING assert not result.app_created assert "app_created" not in cache.call_args.args[2] @@ -1543,7 +1554,11 @@ def test_confirmed_import_reports_creation_only_for_new_apps( monkeypatch.setattr("services.app_dsl_service.redis_client.get", pending_imports.get) monkeypatch.setattr("services.app_dsl_service.redis_client.delete", pending_imports.pop) existing = _app(mode=AppMode.WORKFLOW) - service = AppDslService(unbound_session) + service = AppDslService( + unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ) monkeypatch.setattr(service, "_load_app_for_overwrite", lambda _account, _app_id: existing) monkeypatch.setattr(service, "_validate_workflow_overwrite", lambda _app, _data: None) monkeypatch.setattr(service, "_create_or_update_app", lambda *, app, **_kwargs: app or _app(app_id=str(uuid4()))) diff --git a/api/tests/unit_tests/services/test_app_generate_service.py b/api/tests/unit_tests/services/test_app_generate_service.py index 12f51af15c8c67..edce07410387e1 100644 --- a/api/tests/unit_tests/services/test_app_generate_service.py +++ b/api/tests/unit_tests/services/test_app_generate_service.py @@ -1,10 +1,7 @@ import json import threading import uuid -from collections import defaultdict, deque -from collections.abc import Callable, Generator -from datetime import timedelta -from typing import Any +from collections.abc import Callable from unittest.mock import MagicMock import pytest @@ -12,38 +9,22 @@ from sqlalchemy.orm import Session import services.app_generate_service as ags_module -from core.app.apps.message_based_app_generator import MessageBasedAppGenerator -from core.app.apps.message_generator import MessageGenerator -from core.app.entities.app_invoke_entities import InvokeFrom -from core.app.features.rate_limiting import rate_limit as rate_limit_module -from core.app.features.rate_limiting.rate_limit import RateLimit, RateLimitGenerator -from core.errors.error import AppInvokeQuotaExceededError +from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom from enums import DeploymentEdition, QuotaType from models.account import Account from models.enums import AppStatus from models.model import App, AppMode from models.workflow import Workflow, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot +from services.app.generation.runtime import AppGenerationRuntime from services.app_generate_service import AppGenerateService from services.errors.app import ( TriggerWorkflowServiceModeUnavailableError, WorkflowIdFormatError, WorkflowNotFoundError, ) -from tests.helpers.unit.config_override import apply_config_overrides - -""" -Comprehensive unit tests for services.app_generate_service.AppGenerateService. - -Covers: - - _build_streaming_task_on_subscribe (streams / pubsub / exception / idempotency) - - generate (COMPLETION / AGENT_CHAT / CHAT / ADVANCED_CHAT / WORKFLOW / invalid mode, - streaming & blocking, billing, quota-refund-on-error, rate_limit.exit) - - _get_max_active_requests (all limit combos) - - generate_single_iteration (ADVANCED_CHAT / WORKFLOW / invalid mode) - - generate_single_loop (ADVANCED_CHAT / WORKFLOW / invalid mode) - - generate_more_like_this - - _get_workflow (debugger / non-debugger / specific id / invalid format / not found) -""" +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.variable_contracts import WorkflowExecutionVariables # --------------------------------------------------------------------------- @@ -105,11 +86,11 @@ def _make_user() -> Account: class _RealSessionTest: @pytest.fixture(autouse=True) - def _bind_unbound_session(self, unbound_session: Session) -> None: - self.session = unbound_session + def _bind_session(self, sqlite_session: Session) -> None: + self.session = sqlite_session -def _make_workflow( +def _make_workflow_model( *, workflow_id: str = "workflow-id", created_by: str = "owner-id", @@ -129,42 +110,22 @@ def _make_workflow( "edges": [], } ), - _features={}, + _features="{}", created_by=created_by, _environment_variables=[], _conversation_variables=[], ) -class _RateLimitRedis: - def __init__(self) -> None: - self.values: dict[str, bytes] = {} - self.hashes: dict[str, dict[str, bytes]] = {} - - def exists(self, key: str) -> bool: - return key in self.values or key in self.hashes - - def setex(self, key: str, ttl: timedelta, value: int) -> None: - self.values[key] = str(value).encode() - - def get(self, key: str) -> bytes: - return self.values[key] - - def expire(self, key: str, ttl: timedelta) -> None: - pass - - def hgetall(self, key: str) -> dict[str, bytes]: - return self.hashes.get(key, {}) - - def hlen(self, key: str) -> int: - return len(self.hashes.get(key, {})) - - def hset(self, key: str, field: str, value: str) -> None: - self.hashes.setdefault(key, {})[field] = value.encode() - - def hdel(self, key: str, *fields: str) -> None: - for field in fields: - self.hashes.get(key, {}).pop(field, None) +def _make_workflow( + *, + workflow_id: str = "workflow-id", + created_by: str = "owner-id", + node_types: tuple[str, ...] = (), +) -> WorkflowSnapshot: + return workflow_snapshot( + _make_workflow_model(workflow_id=workflow_id, created_by=created_by, node_types=node_types) + ) # --------------------------------------------------------------------------- @@ -189,7 +150,12 @@ def _unexpected_timer(interval: float, function: Callable[[], bool]) -> _FakeTim raise AssertionError("streams must not create a fallback timer") -class TestAppGenerateServiceCases: +class TestAppGenerateServiceCases(_RealSessionTest): + @pytest.fixture(autouse=True) + def _common_generation(self, mocker: MockerFixture, config_overrides: Callable[..., None]) -> None: + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + mocker.patch("services.app_generate_service.RateLimit", _DummyRateLimit) + class TestBuildStreamingTaskOnSubscribe: def test_streams_starts_only_when_hook_is_invoked_without_creating_timer( self, monkeypatch: pytest.MonkeyPatch, config_overrides: Callable[..., None] @@ -325,431 +291,609 @@ def test_default_active_requests_used_when_app_has_none(self, config_overrides: # --------------------------------------------------------------------------- # generate – every AppMode branch # --------------------------------------------------------------------------- - class TestGenerate(_RealSessionTest): - """Tests for AppGenerateService.generate covering each mode.""" - - @pytest.fixture(autouse=True) - def _common(self, mocker: MockerFixture, config_overrides: Callable[..., None]): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) - mocker.patch("services.app_generate_service.RateLimit", _DummyRateLimit) - - # -- COMPLETION --------------------------------------------------------- - def test_completion_mode(self, mocker: MockerFixture): - gen_spy = mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - return_value={"result": "ok"}, - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) + @pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) + @pytest.mark.parametrize("entry", ["generate", "iteration", "loop"]) + def test_debug_generators_use_injected_loader_and_saver( + self, + monkeypatch: pytest.MonkeyPatch, + workflow_variables: WorkflowExecutionVariables, + mode: AppMode, + entry: str, + *, + workflow_runtime: AppGenerationRuntime, + ) -> None: + from graphon.enums import BuiltinNodeTypes + from services.app.generation.input_adapter import AppInputAdapter + from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + workflow = _make_workflow() + user = _make_user() + monkeypatch.setattr(AppGenerateService, "_get_workflow", lambda *_args, **_kwargs: workflow) + calls: list[str] = [] + + def generate(generator: AppInputAdapter, **_kwargs: object) -> dict[str, str]: + # Keep the real generator constructor and debugger saver selection. + assert generator.draft_variable_loader == workflow_variables.workflow_loader + assert generator.draft_variable_saver == workflow_variables.saver_factory + loader = workflow_variables.workflow_loader(workflow, user.id) + saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) + saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( + process_data=None, outputs={"text": "injected value"} + ) + assert loader.load_variables([["node", "text"]])[0].value == "injected value" + calls.append(entry) + return {"result": "ready"} + + generator_type = WorkflowAppGenerator if mode == AppMode.WORKFLOW else AdvancedChatAppGenerator + method = {"generate": "generate", "iteration": "single_iteration_generate", "loop": "single_loop_generate"}[ + entry + ] + monkeypatch.setattr(generator_type, method, generate) + app = _make_app(mode) + if entry == "generate": result = AppGenerateService.generate( - app_model=_make_app(AppMode.COMPLETION), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, + app_model=app, + user=user, + args={}, + invoke_from=InvokeFrom.DEBUGGER, session=self.session, - ) - assert result == {"result": "ok"} - gen_spy.assert_called_once() - - # -- AGENT_CHAT via mode ------------------------------------------------ - def test_agent_chat_mode(self, mocker: MockerFixture): - gen_spy = mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.generate", - return_value={"result": "agent"}, - ) - mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - result = AppGenerateService.generate( - app_model=_make_app(AppMode.AGENT_CHAT), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, streaming=False, - session=self.session, - ) - assert result == {"result": "agent"} - gen_spy.assert_called_once() - - # -- AGENT_CHAT via is_agent flag (non-AGENT_CHAT mode) ----------------- - def test_agent_via_is_agent_flag(self, mocker: MockerFixture): - gen_spy = mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.generate", - return_value={"result": "agent-via-flag"}, - ) - mocker.patch( - "services.app_generate_service.AgentChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, + variables=workflow_variables, + runtime=workflow_runtime, ) - app = _make_app(AppMode.CHAT) - is_agent = mocker.patch.object(App, "is_agent_with_session", return_value=True) - session = self.session - result = AppGenerateService.generate( + elif entry == "iteration": + result = AppGenerateService.generate_single_iteration( + workflow=workflow, app_model=app, - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, + user=user, + node_id="node", + args={}, streaming=False, - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) - assert result == {"result": "agent-via-flag"} - gen_spy.assert_called_once() - is_agent.assert_called_once_with(session=session) - - # -- AGENT -------------------------------------------------------------- - def test_agent_mode_passes_session(self, mocker: MockerFixture): - gen_spy = mocker.patch( - "services.app_generate_service.AgentAppGenerator.generate", - return_value={"result": "agent"}, - ) - mocker.patch( - "services.app_generate_service.AgentAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - session = self.session - - result = AppGenerateService.generate( - app_model=_make_app(AppMode.AGENT), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=True, - session=session, - ) - - assert result == {"result": "agent"} - assert gen_spy.call_args.kwargs["session"] is session - - # -- CHAT --------------------------------------------------------------- - def test_chat_mode(self, mocker: MockerFixture): - gen_spy = mocker.patch( - "services.app_generate_service.ChatAppGenerator.generate", - return_value={"result": "chat"}, - ) - mocker.patch( - "services.app_generate_service.ChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - app = _make_app(AppMode.CHAT) - result = AppGenerateService.generate( + else: + result = AppGenerateService.generate_single_loop( + workflow=workflow, app_model=app, - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, + user=user, + node_id="node", + args={}, streaming=False, - session=self.session, - ) - assert result == {"result": "chat"} - gen_spy.assert_called_once() - - # -- ADVANCED_CHAT blocking --------------------------------------------- - def test_advanced_chat_blocking(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - - retrieve_spy = mocker.patch("services.app_generate_service.AdvancedChatAppGenerator.retrieve_events") - gen_spy = mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.generate", - return_value={"result": "advanced-blocking"}, - ) - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "ready"} + assert calls == [entry] + + def test_completion_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + gen_spy = mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + return_value={"result": "ok"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + result = AppGenerateService.generate( + app_model=_make_app(AppMode.COMPLETION), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "ok"} + gen_spy.assert_called_once() + + # -- AGENT_CHAT via mode ------------------------------------------------ + def test_agent_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + generator = mocker.patch("services.app_generate_service.AgentChatAppGenerator") + gen_spy = generator.return_value.generate + gen_spy.return_value = {"result": "agent"} + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + result = AppGenerateService.generate( + app_model=_make_app(AppMode.AGENT_CHAT), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "agent"} + gen_spy.assert_called_once() + assert generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker + + # -- AGENT_CHAT via is_agent flag (non-AGENT_CHAT mode) ----------------- + def test_agent_via_is_agent_flag( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + generator = mocker.patch("services.app_generate_service.AgentChatAppGenerator") + gen_spy = generator.return_value.generate + gen_spy.return_value = {"result": "agent-via-flag"} + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + app = _make_app(AppMode.CHAT) + is_agent = mocker.patch.object(App, "is_agent_with_session", return_value=True) + session = self.session + result = AppGenerateService.generate( + app_model=app, + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "agent-via-flag"} + gen_spy.assert_called_once() + assert generator.call_args.kwargs["tool_invoker"] is workflow_runtime.agent_tool_invoker + is_agent.assert_called_once_with(session=session) + + # -- AGENT -------------------------------------------------------------- + def test_agent_mode_passes_session( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + mocker.patch( + "services.app_generate_service.AgentAppGenerator.generate", + return_value={"result": "agent"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + session = self.session + + result = AppGenerateService.generate( + app_model=_make_app(AppMode.AGENT), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=True, + session=session, + variables=workflow_variables, + runtime=workflow_runtime, + ) - session = self.session - result = AppGenerateService.generate( - app_model=_make_app(AppMode.ADVANCED_CHAT), - user=_make_user(), - args={"workflow_id": None, "query": "hi", "inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=session, - ) - assert result == {"result": "advanced-blocking"} - call_kwargs = gen_spy.call_args.kwargs - assert call_kwargs.get("streaming") is False - assert call_kwargs["session"] is session - retrieve_spy.assert_not_called() - - # -- ADVANCED_CHAT streaming -------------------------------------------- - def test_advanced_chat_streaming(self, mocker: MockerFixture, config_overrides: Callable[..., None]): - config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AppExecutionParams.new", - return_value=MagicMock(workflow_run_id="wfr-1", model_dump_json=MagicMock(return_value="{}")), - ) - delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay") - retrieve_spy = mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.retrieve_events", - return_value=iter([]), - ) + assert result == {"result": "agent"} + + # -- CHAT --------------------------------------------------------------- + def test_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + gen_spy = mocker.patch( + "services.app_generate_service.ChatAppGenerator.generate", + return_value={"result": "chat"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + app = _make_app(AppMode.CHAT) + result = AppGenerateService.generate( + app_model=app, + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "chat"} + gen_spy.assert_called_once() + + # -- ADVANCED_CHAT blocking --------------------------------------------- + def test_advanced_chat_blocking( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + + retrieve_spy = mocker.patch("services.app_generate_service.WorkflowEventStream.retrieve_events") + gen_spy = mocker.patch( + "services.app_generate_service.AdvancedChatAppGenerator.generate", + return_value={"result": "advanced-blocking"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) - AppGenerateService.generate( - app_model=_make_app(AppMode.ADVANCED_CHAT), - user=_make_user(), - args={"workflow_id": None, "query": "hi", "inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=True, - session=self.session, - ) - # In streaming mode it should go through retrieve_events, not generate - retrieve_spy.assert_called_once() - # Dispatch is gated on subscribe; simulate the SSE layer entering the - # subscription, which is what actually invokes on_subscribe. - on_subscribe = retrieve_spy.call_args.kwargs["on_subscribe"] - on_subscribe() - delay_spy.assert_called_once() - - # -- WORKFLOW blocking -------------------------------------------------- - def test_workflow_blocking(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - gen_spy = mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.generate", - return_value={"result": "workflow-blocking"}, - ) - mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) + session = self.session + result = AppGenerateService.generate( + app_model=_make_app(AppMode.ADVANCED_CHAT), + user=_make_user(), + args={"workflow_id": None, "query": "hi", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "advanced-blocking"} + call_kwargs = gen_spy.call_args.kwargs + assert call_kwargs.get("streaming") is False + assert "session" not in call_kwargs + retrieve_spy.assert_not_called() + + # -- ADVANCED_CHAT streaming -------------------------------------------- + def test_advanced_chat_streaming( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.AppExecutionParams.new", + return_value=MagicMock(workflow_run_id="wfr-1", model_dump_json=MagicMock(return_value="{}")), + ) + delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay") + retrieve_spy = mocker.patch( + "services.app_generate_service.WorkflowEventStream.retrieve_events", + return_value=iter([]), + ) - session = self.session - result = AppGenerateService.generate( - app_model=_make_app(AppMode.WORKFLOW), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=session, - ) - assert result == {"result": "workflow-blocking"} - call_kwargs = gen_spy.call_args.kwargs - assert call_kwargs.get("pause_state_config") is not None - assert call_kwargs["pause_state_config"].state_owner_user_id == "owner-id" - - @pytest.mark.parametrize( - "invoke_from", - [InvokeFrom.OPENAPI, InvokeFrom.SERVICE_API, InvokeFrom.WEB_APP], + AppGenerateService.generate( + app_model=_make_app(AppMode.ADVANCED_CHAT), + user=_make_user(), + args={"workflow_id": None, "query": "hi", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=True, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + # In streaming mode it should go through retrieve_events, not generate + retrieve_spy.assert_called_once() + # Dispatch is gated on subscribe; simulate the SSE layer entering the + # subscription, which is what actually invokes on_subscribe. + on_subscribe = retrieve_spy.call_args.kwargs["on_subscribe"] + on_subscribe() + delay_spy.assert_called_once() + + # -- WORKFLOW blocking -------------------------------------------------- + def test_workflow_blocking( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + gen_spy = mocker.patch( + "services.app_generate_service.WorkflowAppGenerator.generate", + return_value={"result": "workflow-blocking"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, ) - @pytest.mark.parametrize("node_type", ["trigger-plugin", "trigger-schedule", "trigger-webhook"]) - def test_trigger_workflow_rejects_manual_service_surfaces( - self, - invoke_from: InvokeFrom, - node_type: str, - mocker: MockerFixture, - ) -> None: - workflow = _make_workflow(node_types=(node_type,)) - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - generate = mocker.patch("services.app_generate_service.WorkflowAppGenerator.generate") - - with pytest.raises(TriggerWorkflowServiceModeUnavailableError): - AppGenerateService.generate( - app_model=_make_app(AppMode.WORKFLOW), - user=_make_user(), - args={"inputs": {}}, - invoke_from=invoke_from, - streaming=False, - session=self.session, - ) - - generate.assert_not_called() - - def test_trigger_workflow_allows_trigger_execution(self, mocker: MockerFixture) -> None: - workflow = _make_workflow(node_types=("trigger-webhook",)) - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - generate = mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.generate", - return_value={"result": "trigger"}, - ) - mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", - side_effect=lambda value: value, - ) - result = AppGenerateService.generate( + session = self.session + result = AppGenerateService.generate( + app_model=_make_app(AppMode.WORKFLOW), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"result": "workflow-blocking"} + call_kwargs = gen_spy.call_args.kwargs + assert call_kwargs.get("pause_state_config") is not None + assert call_kwargs["pause_state_config"].state_owner_user_id == "owner-id" + assert call_kwargs["execution_identity"].user_id == "user-id" + assert call_kwargs["execution_identity"].user_from == UserFrom.ACCOUNT + + @pytest.mark.parametrize( + "invoke_from", + [InvokeFrom.OPENAPI, InvokeFrom.SERVICE_API, InvokeFrom.WEB_APP], + ) + @pytest.mark.parametrize("node_type", ["trigger-plugin", "trigger-schedule", "trigger-webhook"]) + def test_trigger_workflow_rejects_manual_service_surfaces( + self, + invoke_from: InvokeFrom, + node_type: str, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ) -> None: + workflow = _make_workflow(node_types=(node_type,)) + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + generate = mocker.patch("services.app_generate_service.WorkflowAppGenerator.generate") + + with pytest.raises(TriggerWorkflowServiceModeUnavailableError): + AppGenerateService.generate( app_model=_make_app(AppMode.WORKFLOW), user=_make_user(), args={"inputs": {}}, - invoke_from=InvokeFrom.TRIGGER, + invoke_from=invoke_from, streaming=False, session=self.session, - ) + variables=workflow_variables, + runtime=workflow_runtime, + ) + + generate.assert_not_called() + + def test_trigger_workflow_allows_trigger_execution( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ) -> None: + workflow = _make_workflow(node_types=("trigger-webhook",)) + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + generate = mocker.patch( + "services.app_generate_service.WorkflowAppGenerator.generate", + return_value={"result": "trigger"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda value: value, + ) - assert result == {"result": "trigger"} - generate.assert_called_once() + result = AppGenerateService.generate( + app_model=_make_app(AppMode.WORKFLOW), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.TRIGGER, + streaming=False, + session=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, + ) - def test_specific_start_workflow_version_remains_runnable(self, mocker: MockerFixture) -> None: - workflow_id = str(uuid.uuid4()) - workflow = _make_workflow(workflow_id=workflow_id, node_types=("start",)) - get_workflow = mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.generate", - return_value={"result": "version"}, - ) - mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", - side_effect=lambda value: value, - ) - app = _make_app(AppMode.WORKFLOW) - session = self.session + assert result == {"result": "trigger"} + generate.assert_called_once() + + def test_specific_start_workflow_version_remains_runnable( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ) -> None: + workflow_id = str(uuid.uuid4()) + workflow = _make_workflow(workflow_id=workflow_id, node_types=("start",)) + get_workflow = mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.WorkflowAppGenerator.generate", + return_value={"result": "version"}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda value: value, + ) + app = _make_app(AppMode.WORKFLOW) + session = MagicMock() + + result = AppGenerateService.generate( + app_model=app, + user=_make_user(), + args={"inputs": {}, "workflow_id": workflow_id}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=session, + variables=workflow_variables, + runtime=workflow_runtime, + ) - result = AppGenerateService.generate( + assert result == {"result": "version"} + get_workflow.assert_called_once_with(app, InvokeFrom.SERVICE_API, workflow_id, session=session, snapshot=None) + + # -- WORKFLOW streaming ------------------------------------------------- + def test_workflow_streaming( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.AppExecutionParams.new", + return_value=MagicMock(workflow_run_id="wfr-2", model_dump_json=MagicMock(return_value="{}")), + ) + delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay") + retrieve_spy = mocker.patch( + "services.app_generate_service.WorkflowEventStream.retrieve_events", + return_value=iter([]), + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + + AppGenerateService.generate( + app_model=_make_app(AppMode.WORKFLOW), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=True, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + retrieve_spy.assert_called_once() + # Dispatch is gated on subscribe; simulate the SSE layer entering the + # subscription, which is what actually invokes on_subscribe. + on_subscribe = retrieve_spy.call_args.kwargs["on_subscribe"] + on_subscribe() + delay_spy.assert_called_once() + + # -- Invalid mode ------------------------------------------------------- + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + app = _make_app("invalid-mode") + with pytest.raises(ValueError, match="Invalid app mode"): + AppGenerateService.generate( app_model=app, user=_make_user(), - args={"inputs": {}, "workflow_id": workflow_id}, + args={"inputs": {}}, invoke_from=InvokeFrom.SERVICE_API, streaming=False, - session=session, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) - assert result == {"result": "version"} - get_workflow.assert_called_once_with(app, InvokeFrom.SERVICE_API, workflow_id, session=session) - # -- WORKFLOW streaming ------------------------------------------------- - def test_workflow_streaming(self, mocker: MockerFixture, config_overrides: Callable[..., None]): - config_overrides(PUBSUB_REDIS_CHANNEL_TYPE="streams") - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AppExecutionParams.new", - return_value=MagicMock(workflow_run_id="wfr-2", model_dump_json=MagicMock(return_value="{}")), - ) - delay_spy = mocker.patch("services.app_generate_service.workflow_based_app_execution_task.delay") - retrieve_spy = mocker.patch( - "services.app_generate_service.MessageBasedAppGenerator.retrieve_events", - return_value=iter([]), - ) - mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) +# --------------------------------------------------------------------------- +# generate – billing / quota +# --------------------------------------------------------------------------- +class TestGenerateBilling(_RealSessionTest): + @pytest.fixture(autouse=True) + def _common(self, mocker: MockerFixture): + mocker.patch("services.app_generate_service.RateLimit", _DummyRateLimit) + + def test_cloud_edition_consumes_quota( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + quota_charge = MagicMock() + reserve_mock = mocker.patch( + "services.app_generate_service.QuotaService.reserve", + return_value=quota_charge, + ) + mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + return_value={"ok": True}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + AppGenerateService.generate( + app_model=_make_app(AppMode.COMPLETION), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + reserve_mock.assert_called_once_with(QuotaType.WORKFLOW, "tenant-id") + quota_charge.commit.assert_called_once() + + def test_billing_quota_exceeded_raises_rate_limit_error( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + from services.errors.app import QuotaExceededError + from services.errors.llm import InvokeRateLimitError + + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + mocker.patch( + "services.app_generate_service.QuotaService.reserve", + side_effect=QuotaExceededError(feature="workflow", tenant_id="t", required=1), + ) + + with pytest.raises(InvokeRateLimitError): AppGenerateService.generate( - app_model=_make_app(AppMode.WORKFLOW), + app_model=_make_app(AppMode.COMPLETION), user=_make_user(), args={"inputs": {}}, invoke_from=InvokeFrom.SERVICE_API, - streaming=True, + streaming=False, session=self.session, - ) - retrieve_spy.assert_called_once() - # Dispatch is gated on subscribe; simulate the SSE layer entering the - # subscription, which is what actually invokes on_subscribe. - on_subscribe = retrieve_spy.call_args.kwargs["on_subscribe"] - on_subscribe() - delay_spy.assert_called_once() - - # -- Invalid mode ------------------------------------------------------- - def test_invalid_mode_raises(self, mocker: MockerFixture): - app = _make_app("invalid-mode") - with pytest.raises(ValueError, match="Invalid app mode"): - AppGenerateService.generate( - app_model=app, - user=_make_user(), - args={}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=self.session, - ) - - class TestStreamingRateLimit(_RealSessionTest): - @pytest.mark.parametrize("app_mode", [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) - @pytest.mark.parametrize("termination", ["finish", "error", "close"]) - def test_single_slot_is_held_until_stream_ends( - self, - monkeypatch: pytest.MonkeyPatch, - config_overrides: Callable[..., None], - app_mode: AppMode, - termination: str, - ) -> None: - config_overrides( - DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY, - APP_MAX_ACTIVE_REQUESTS=0, - APP_DEFAULT_ACTIVE_REQUESTS=0, - PUBSUB_REDIS_CHANNEL_TYPE="streams", - ) - redis = _RateLimitRedis() - monkeypatch.setattr(rate_limit_module, "redis_client", redis) - monkeypatch.setattr(RateLimit, "_instance_dict", {}) - workflow = _make_workflow() - monkeypatch.setattr(AppGenerateService, "_get_workflow", lambda *_args, **_kwargs: workflow) - - def retrieve_events(*_args: object, **_kwargs: object) -> Generator[str]: - yield "ping" - if termination == "error": - raise RuntimeError("stream failed") - - monkeypatch.setattr(MessageBasedAppGenerator, "retrieve_events", retrieve_events) - app = _make_app(app_mode, max_active_requests=1) - - def generate() -> RateLimitGenerator: - response = AppGenerateService.generate( - app_model=app, - user=_make_user(), - args={"inputs": {}, "query": "hi"}, - invoke_from=InvokeFrom.DEBUGGER, - streaming=True, - session=self.session, - ) - assert isinstance(response, RateLimitGenerator) - return response - - first = generate() - active_key = first.rate_limit.active_requests_key - assert set(redis.hgetall(active_key)) == {first.request_id} - - with pytest.raises(AppInvokeQuotaExceededError, match="maximum concurrent requests allowed"): - generate() - assert set(redis.hgetall(active_key)) == {first.request_id} - - assert next(first) == "event: ping\n\n" - if termination == "finish": - assert list(first) == [] - elif termination == "error": - with pytest.raises(RuntimeError, match="stream failed"): - next(first) - else: - first.close() - first.close() - assert redis.hlen(active_key) == 0 - - following = generate() - assert following.request_id != first.request_id - assert set(redis.hgetall(active_key)) == {following.request_id} - following.close() - assert redis.hlen(active_key) == 0 - - # --------------------------------------------------------------------------- - # generate – billing / quota - # --------------------------------------------------------------------------- - class TestGenerateBilling(_RealSessionTest): - @pytest.fixture(autouse=True) - def _common(self, mocker: MockerFixture): - mocker.patch("services.app_generate_service.RateLimit", _DummyRateLimit) - - def test_cloud_edition_consumes_quota(self, mocker: MockerFixture, config_overrides: Callable[..., None]): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - quota_charge = MagicMock() - reserve_mock = mocker.patch( - "services.app_generate_service.QuotaService.reserve", - return_value=quota_charge, - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - return_value={"ok": True}, - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) + variables=workflow_variables, + runtime=workflow_runtime, + ) + + def test_exception_refunds_quota_and_exits_rate_limit( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + quota_charge = MagicMock() + mocker.patch( + "services.app_generate_service.QuotaService.reserve", + return_value=quota_charge, + ) + mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + side_effect=RuntimeError("boom"), + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + with pytest.raises(RuntimeError, match="boom"): AppGenerateService.generate( app_model=_make_app(AppMode.COMPLETION), user=_make_user(), @@ -757,83 +901,78 @@ def test_cloud_edition_consumes_quota(self, mocker: MockerFixture, config_overri invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, - ) - reserve_mock.assert_called_once_with(QuotaType.WORKFLOW, "tenant-id") - quota_charge.commit.assert_called_once() - - def test_billing_quota_exceeded_raises_rate_limit_error( - self, mocker: MockerFixture, config_overrides: Callable[..., None] - ): - from services.errors.app import QuotaExceededError - from services.errors.llm import InvokeRateLimitError - - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - mocker.patch( - "services.app_generate_service.QuotaService.reserve", - side_effect=QuotaExceededError(feature="workflow", tenant_id="t", required=1), - ) - - with pytest.raises(InvokeRateLimitError): - AppGenerateService.generate( - app_model=_make_app(AppMode.COMPLETION), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=self.session, - ) - - def test_exception_refunds_quota_and_exits_rate_limit( - self, mocker: MockerFixture, config_overrides: Callable[..., None] - ): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - quota_charge = MagicMock() - mocker.patch( - "services.app_generate_service.QuotaService.reserve", - return_value=quota_charge, - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - side_effect=RuntimeError("boom"), - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - - with pytest.raises(RuntimeError, match="boom"): - AppGenerateService.generate( - app_model=_make_app(AppMode.COMPLETION), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=self.session, - ) - quota_charge.refund.assert_called_once() - - def test_rate_limit_exit_called_in_finally_for_blocking( - self, mocker: MockerFixture, config_overrides: Callable[..., None] - ): - """For non-streaming (blocking) calls, rate_limit.exit should be called in finally.""" - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + variables=workflow_variables, + runtime=workflow_runtime, + ) + quota_charge.refund.assert_called_once() + + def test_rate_limit_exit_called_in_finally_for_blocking( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + """For non-streaming (blocking) calls, rate_limit.exit should be called in finally.""" + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + + exit_calls: list[str] = [] + + class _TrackingRateLimit(_DummyRateLimit): + def exit(self, request_id: str) -> None: + exit_calls.append(request_id) + + mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) + mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + return_value={"ok": True}, + ) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) - exit_calls: list[str] = [] + AppGenerateService.generate( + app_model=_make_app(AppMode.COMPLETION), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + # exit is called in finally block for non-streaming + assert exit_calls == ["dummy-request-id"] + + def test_blocking_failure_exits_rate_limit_once( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + quota_charge = MagicMock() + mocker.patch( + "services.app_generate_service.QuotaService.reserve", + return_value=quota_charge, + ) + exit_calls: list[str] = [] - class _TrackingRateLimit(_DummyRateLimit): - def exit(self, request_id: str) -> None: - exit_calls.append(request_id) + class _TrackingRateLimit(_DummyRateLimit): + def exit(self, request_id: str) -> None: + exit_calls.append(request_id) - mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - return_value={"ok": True}, - ) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) + mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) + mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + side_effect=RuntimeError("boom"), + ) + with pytest.raises(RuntimeError, match="boom"): AppGenerateService.generate( app_model=_make_app(AppMode.COMPLETION), user=_make_user(), @@ -841,488 +980,296 @@ def exit(self, request_id: str) -> None: invoke_from=InvokeFrom.SERVICE_API, streaming=False, session=self.session, - ) - # exit is called in finally block for non-streaming - assert exit_calls == ["dummy-request-id"] - - def test_blocking_failure_exits_rate_limit_once( - self, mocker: MockerFixture, config_overrides: Callable[..., None] - ): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - quota_charge = MagicMock() - mocker.patch( - "services.app_generate_service.QuotaService.reserve", - return_value=quota_charge, - ) - exit_calls: list[str] = [] + variables=workflow_variables, + runtime=workflow_runtime, + ) + + quota_charge.refund.assert_called_once() + assert exit_calls == ["dummy-request-id"] + + def test_streaming_failure_exits_rate_limit_once( + self, + mocker: MockerFixture, + config_overrides: Callable[..., None], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) + quota_charge = MagicMock() + mocker.patch( + "services.app_generate_service.QuotaService.reserve", + return_value=quota_charge, + ) + exit_calls: list[str] = [] - class _TrackingRateLimit(_DummyRateLimit): - def exit(self, request_id: str) -> None: - exit_calls.append(request_id) + class _TrackingRateLimit(_DummyRateLimit): + def exit(self, request_id: str) -> None: + exit_calls.append(request_id) - mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - side_effect=RuntimeError("boom"), - ) - - with pytest.raises(RuntimeError, match="boom"): - AppGenerateService.generate( - app_model=_make_app(AppMode.COMPLETION), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=False, - session=self.session, - ) - - quota_charge.refund.assert_called_once() - assert exit_calls == ["dummy-request-id"] - - def test_streaming_failure_exits_rate_limit_once( - self, mocker: MockerFixture, config_overrides: Callable[..., None] - ): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - quota_charge = MagicMock() - mocker.patch( - "services.app_generate_service.QuotaService.reserve", - return_value=quota_charge, - ) - exit_calls: list[str] = [] - - class _TrackingRateLimit(_DummyRateLimit): - def exit(self, request_id: str) -> None: - exit_calls.append(request_id) + mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) + mocker.patch( + "services.app_generate_service.CompletionAppGenerator.generate", + side_effect=RuntimeError("boom"), + ) - mocker.patch("services.app_generate_service.RateLimit", _TrackingRateLimit) - mocker.patch( - "services.app_generate_service.CompletionAppGenerator.generate", - side_effect=RuntimeError("boom"), + with pytest.raises(RuntimeError, match="boom"): + AppGenerateService.generate( + app_model=_make_app(AppMode.COMPLETION), + user=_make_user(), + args={"inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=True, + session=self.session, + variables=workflow_variables, + runtime=workflow_runtime, ) - with pytest.raises(RuntimeError, match="boom"): - AppGenerateService.generate( - app_model=_make_app(AppMode.COMPLETION), - user=_make_user(), - args={"inputs": {}}, - invoke_from=InvokeFrom.SERVICE_API, - streaming=True, - session=self.session, - ) - - quota_charge.refund.assert_called_once() - assert exit_calls == ["dummy-request-id"] - # --------------------------------------------------------------------------- - # _get_workflow - # --------------------------------------------------------------------------- - class TestGetWorkflow(_RealSessionTest): - def test_debugger_fetches_draft(self, mocker: MockerFixture): - draft_wf = _make_workflow() - ws = MagicMock() - ws.get_draft_workflow.return_value = draft_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session +# --------------------------------------------------------------------------- +# _get_workflow +# --------------------------------------------------------------------------- +class TestGetWorkflow(_RealSessionTest): + def test_debugger_fetches_draft(self): + workflow = _make_workflow_model() + self.session.add(workflow) + self.session.commit() + result = AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session + ) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.tenant_id, result.app_id, result.version) == ( + workflow.id, + workflow.tenant_id, + workflow.app_id, + "draft", + ) + assert result.graph == workflow.graph + + def test_debugger_raises_when_no_draft(self): + with pytest.raises(ValueError, match="Workflow not initialized"): + AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session) + + def test_non_debugger_fetches_published(self): + workflow = _make_workflow_model() + workflow.version = "published" + self.session.add(workflow) + self.session.commit() + app = _make_app(AppMode.WORKFLOW) + app.workflow_id = workflow.id + result = AppGenerateService._get_workflow(app, InvokeFrom.SERVICE_API, session=self.session) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.version) == (workflow.id, "published") + assert result.graph == workflow.graph + + def test_non_debugger_raises_when_no_published(self): + with pytest.raises(ValueError, match="Workflow not published"): + AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, session=self.session) + + def test_specific_workflow_id_valid_uuid(self): + workflow = _make_workflow_model(workflow_id=str(uuid.uuid4())) + workflow.version = "published" + self.session.add(workflow) + self.session.commit() + result = AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session + ) + assert isinstance(result, WorkflowSnapshot) + assert (result.id, result.version) == (workflow.id, "published") + assert result.graph == workflow.graph + + def test_specific_workflow_id_invalid_uuid(self): + with pytest.raises(WorkflowIdFormatError): + AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id="not-a-uuid", session=self.session + ) + + @pytest.mark.parametrize("owner", ["missing", "other-tenant", "other-app"]) + def test_specific_workflow_id_not_found(self, owner: str): + workflow = _make_workflow_model(workflow_id=str(uuid.uuid4())) + workflow.version = "published" + if owner != "missing": + if owner == "other-tenant": + workflow.tenant_id = owner + else: + workflow.app_id = owner + self.session.add(workflow) + self.session.commit() + with pytest.raises(WorkflowNotFoundError): + AppGenerateService._get_workflow( + _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, workflow_id=workflow.id, session=self.session ) - assert result is draft_wf - ws.get_draft_workflow.assert_called_once() - - def test_debugger_raises_when_no_draft(self, mocker: MockerFixture): - ws = MagicMock() - ws.get_draft_workflow.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - with pytest.raises(ValueError, match="Workflow not initialized"): - AppGenerateService._get_workflow(_make_app(AppMode.WORKFLOW), InvokeFrom.DEBUGGER, session=self.session) - def test_non_debugger_fetches_published(self, mocker: MockerFixture): - pub_wf = _make_workflow() - ws = MagicMock() - ws.get_published_workflow.return_value = pub_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, session=self.session - ) - assert result is pub_wf - ws.get_published_workflow.assert_called_once() - - def test_non_debugger_raises_when_no_published(self, mocker: MockerFixture): - ws = MagicMock() - ws.get_published_workflow.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - with pytest.raises(ValueError, match="Workflow not published"): - AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), InvokeFrom.SERVICE_API, session=self.session - ) - - def test_specific_workflow_id_valid_uuid(self, mocker: MockerFixture): - valid_uuid = str(uuid.uuid4()) - specific_wf = _make_workflow(workflow_id=valid_uuid) - ws = MagicMock() - ws.get_published_workflow_by_id.return_value = specific_wf - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - result = AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id=valid_uuid, - session=self.session, - ) - assert result is specific_wf - ws.get_published_workflow_by_id.assert_called_once() - - def test_specific_workflow_id_invalid_uuid(self, mocker: MockerFixture): - ws = MagicMock() - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - with pytest.raises(WorkflowIdFormatError): - AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id="not-a-uuid", - session=self.session, - ) - - def test_specific_workflow_id_not_found(self, mocker: MockerFixture): - valid_uuid = str(uuid.uuid4()) - ws = MagicMock() - ws.get_published_workflow_by_id.return_value = None - mocker.patch("services.app_generate_service.WorkflowService", return_value=ws) - - with pytest.raises(WorkflowNotFoundError): - AppGenerateService._get_workflow( - _make_app(AppMode.WORKFLOW), - InvokeFrom.SERVICE_API, - workflow_id=valid_uuid, - session=self.session, - ) - - # --------------------------------------------------------------------------- - # generate_single_iteration - # --------------------------------------------------------------------------- - class TestGenerateSingleIteration(_RealSessionTest): - def test_advanced_chat_mode(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - iter_spy = mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.single_iteration_generate", - return_value={"event": "iteration"}, - ) - app = _make_app(AppMode.ADVANCED_CHAT) - session = self.session - result = AppGenerateService.generate_single_iteration( - app_model=app, - user=_make_user(), - node_id="n1", - args={"k": "v"}, - session=session, - ) - iter_spy.assert_called_once() - assert iter_spy.call_args.kwargs["session"] is session - assert result == {"event": "iteration"} - - def test_workflow_mode(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - iter_spy = mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.single_iteration_generate", - return_value={"event": "wf-iteration"}, - ) - app = _make_app(AppMode.WORKFLOW) - session = self.session - result = AppGenerateService.generate_single_iteration( +# --------------------------------------------------------------------------- +# generate_single_iteration +# --------------------------------------------------------------------------- +class TestGenerateSingleIteration(_RealSessionTest): + def test_advanced_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + iter_spy = mocker.patch( + "services.app_generate_service.AdvancedChatAppGenerator.single_iteration_generate", + return_value={"event": "iteration"}, + ) + app = _make_app(AppMode.ADVANCED_CHAT) + result = AppGenerateService.generate_single_iteration( + workflow=workflow, + app_model=app, + user=_make_user(), + node_id="n1", + args={"k": "v"}, + variables=workflow_variables, + runtime=workflow_runtime, + ) + iter_spy.assert_called_once() + assert result == {"event": "iteration"} + + def test_workflow_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + iter_spy = mocker.patch( + "services.app_generate_service.WorkflowAppGenerator.single_iteration_generate", + return_value={"event": "wf-iteration"}, + ) + app = _make_app(AppMode.WORKFLOW) + result = AppGenerateService.generate_single_iteration( + workflow=workflow, + app_model=app, + user=_make_user(), + node_id="n1", + args={"k": "v"}, + variables=workflow_variables, + runtime=workflow_runtime, + ) + iter_spy.assert_called_once() + assert "session" not in iter_spy.call_args.kwargs + assert iter_spy.call_args.kwargs["execution_identity"].user_id == "user-id" + assert iter_spy.call_args.kwargs["execution_identity"].user_from == UserFrom.ACCOUNT + assert result == {"event": "wf-iteration"} + + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + app = _make_app(AppMode.CHAT) + with pytest.raises(ValueError, match="Invalid app mode"): + AppGenerateService.generate_single_iteration( + workflow=_make_workflow(), app_model=app, user=_make_user(), node_id="n1", - args={"k": "v"}, - session=session, + args={}, + variables=workflow_variables, + runtime=workflow_runtime, ) - iter_spy.assert_called_once() - assert iter_spy.call_args.kwargs["session"] is session - assert result == {"event": "wf-iteration"} - def test_invalid_mode_raises(self, mocker: MockerFixture): - app = _make_app(AppMode.CHAT) - with pytest.raises(ValueError, match="Invalid app mode"): - AppGenerateService.generate_single_iteration( - app_model=app, user=_make_user(), node_id="n1", args={}, session=self.session - ) - # --------------------------------------------------------------------------- - # generate_single_loop - # --------------------------------------------------------------------------- - class TestGenerateSingleLoop(_RealSessionTest): - def test_advanced_chat_mode(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - loop_spy = mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.single_loop_generate", - return_value={"event": "loop"}, - ) - app = _make_app(AppMode.ADVANCED_CHAT) - session = self.session - result = AppGenerateService.generate_single_loop( - app_model=app, - user=_make_user(), - node_id="n1", - args=MagicMock(), - session=session, - ) - loop_spy.assert_called_once() - assert loop_spy.call_args.kwargs["session"] is session - assert result == {"event": "loop"} - - def test_workflow_mode(self, mocker: MockerFixture): - workflow = _make_workflow() - mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) - mocker.patch( - "services.app_generate_service.AdvancedChatAppGenerator.convert_to_event_stream", - side_effect=lambda x: x, - ) - loop_spy = mocker.patch( - "services.app_generate_service.WorkflowAppGenerator.single_loop_generate", - return_value={"event": "wf-loop"}, - ) - app = _make_app(AppMode.WORKFLOW) - session = self.session - result = AppGenerateService.generate_single_loop( +# --------------------------------------------------------------------------- +# generate_single_loop +# --------------------------------------------------------------------------- +class TestGenerateSingleLoop(_RealSessionTest): + def test_advanced_chat_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + loop_spy = mocker.patch( + "services.app_generate_service.AdvancedChatAppGenerator.single_loop_generate", + return_value={"event": "loop"}, + ) + app = _make_app(AppMode.ADVANCED_CHAT) + result = AppGenerateService.generate_single_loop( + workflow=workflow, + app_model=app, + user=_make_user(), + node_id="n1", + args=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, + ) + loop_spy.assert_called_once() + assert result == {"event": "loop"} + + def test_workflow_mode( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + workflow = _make_workflow() + mocker.patch.object(AppGenerateService, "_get_workflow", return_value=workflow) + mocker.patch( + "services.app_generate_service.convert_to_event_stream", + side_effect=lambda x: x, + ) + loop_spy = mocker.patch( + "services.app_generate_service.WorkflowAppGenerator.single_loop_generate", + return_value={"event": "wf-loop"}, + ) + app = _make_app(AppMode.WORKFLOW) + result = AppGenerateService.generate_single_loop( + workflow=workflow, + app_model=app, + user=_make_user(), + node_id="n1", + args=MagicMock(), + variables=workflow_variables, + runtime=workflow_runtime, + ) + loop_spy.assert_called_once() + assert "session" not in loop_spy.call_args.kwargs + assert loop_spy.call_args.kwargs["execution_identity"].user_id == "user-id" + assert loop_spy.call_args.kwargs["execution_identity"].user_from == UserFrom.ACCOUNT + assert result == {"event": "wf-loop"} + + def test_invalid_mode_raises( + self, + mocker: MockerFixture, + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: AppGenerationRuntime, + ): + app = _make_app(AppMode.COMPLETION) + with pytest.raises(ValueError, match="Invalid app mode"): + AppGenerateService.generate_single_loop( + workflow=_make_workflow(), app_model=app, user=_make_user(), node_id="n1", args=MagicMock(), - session=session, + variables=workflow_variables, + runtime=workflow_runtime, ) - loop_spy.assert_called_once() - assert loop_spy.call_args.kwargs["session"] is session - assert result == {"event": "wf-loop"} - - def test_invalid_mode_raises(self, mocker: MockerFixture): - app = _make_app(AppMode.COMPLETION) - with pytest.raises(ValueError, match="Invalid app mode"): - AppGenerateService.generate_single_loop( - app_model=app, user=_make_user(), node_id="n1", args=MagicMock(), session=self.session - ) - - # --------------------------------------------------------------------------- - # generate_more_like_this - # --------------------------------------------------------------------------- - - -# ----------------------------- -# Fakes for Redis Pub/Sub flow -# ----------------------------- -class _FakePubSub: - def __init__(self, store: dict[str, deque[bytes]]): - self._store = store - self._subs: set[str] = set() - self._closed = False - - def subscribe(self, topic: str) -> None: - self._subs.add(topic) - - def unsubscribe(self, topic: str) -> None: - self._subs.discard(topic) - - def close(self) -> None: - self._closed = True - - def get_message(self, ignore_subscribe_messages: bool = True, timeout: int | float | None = 1): - # simulate a non-blocking poll; return first available - if self._closed: - return None - for t in list(self._subs): - q = self._store.get(t) - if q and len(q) > 0: - payload = q.popleft() - return {"type": "message", "channel": t, "data": payload} - # no message - return None - - -class _FakeRedisClient: - def __init__(self, store: dict[str, deque[bytes]]): - self._store = store - - def pubsub(self): - return _FakePubSub(self._store) - - def publish(self, topic: str, payload: bytes) -> None: - self._store.setdefault(topic, deque()).append(payload) - - -# ------------------------------------ -# Fakes for Redis Streams (XADD/XREAD) -# ------------------------------------ -class _FakeStreams: - def __init__(self) -> None: - # key -> list[(id, {field: value})] - self._data: dict[str, list[tuple[str, dict]]] = defaultdict(list) - self._seq: dict[str, int] = defaultdict(int) - # Mirrors real Redis: "$" resolves once, to the tail of the stream at the moment - # the first xread naming it is issued -- entries added earlier are never seen. - self._dollar_snapshots: dict[str, int] = {} - - def xadd(self, key: str, fields: dict[str, Any], *, maxlen: int | None = None) -> str: - # maxlen is accepted for API compatibility with redis-py; ignored in this test double - self._seq[key] += 1 - eid = f"{self._seq[key]}-0" - self._data[key].append((eid, fields)) - return eid - - def expire(self, key: str, seconds: int) -> None: - # no-op for tests - return None - - def xrevrange(self, key: str, count: int | None = None): - entries = list(reversed(self._data.get(key, []))) - if count is not None: - entries = entries[:count] - return entries - - def xread(self, streams: dict[str, Any], block: int | None = None, count: int | None = None): - assert len(streams) == 1 - key, last_id = next(iter(streams.items())) - entries = self._data.get(key, []) - start = 0 - if last_id == "$": - start = self._dollar_snapshots.setdefault(key, len(entries)) - elif last_id != "0-0": - for i, (eid, _f) in enumerate(entries): - if eid == last_id: - start = i + 1 - break - if start >= len(entries): - return [] - end = len(entries) if count is None else min(len(entries), start + count) - return [(key, entries[start:end])] - - -def _publish_events(app_mode: AppMode, run_id: str, events: list[dict]): - # Publish events to the same topic used by MessageGenerator - topic = MessageGenerator.get_response_topic(app_mode, run_id) - for ev in events: - topic.publish(json.dumps(ev).encode()) - - -class TestAppGenerateServiceStreamingIntegrationCases: - @staticmethod - @pytest.fixture - def _patch_get_channel_streams(monkeypatch: pytest.MonkeyPatch): - from libs.broadcast_channel.redis.streams_channel import StreamsBroadcastChannel - - fake = _FakeStreams() - chan = StreamsBroadcastChannel(fake, retention_seconds=60) - - def _get_channel(): - return chan - - # Patch the source and the imported aliases used by MessageGenerator / MessageBasedAppGenerator - monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_generator.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_based_app_generator.get_pubsub_broadcast_channel", lambda: chan) - apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") - - @staticmethod - @pytest.fixture - def _patch_get_channel_pubsub(monkeypatch: pytest.MonkeyPatch): - from libs.broadcast_channel.redis.pubsub_channel import BroadcastChannel as RedisBroadcastChannel - - store: dict[str, deque[bytes]] = defaultdict(deque) - client = _FakeRedisClient(store) - chan = RedisBroadcastChannel(client) - - def _get_channel(): - return chan - - # Patch both the source and the imported alias used by MessageGenerator - monkeypatch.setattr("extensions.ext_redis.get_pubsub_broadcast_channel", lambda: chan) - monkeypatch.setattr("core.app.apps.message_generator.get_pubsub_broadcast_channel", lambda: chan) - apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="pubsub") - - @staticmethod - @pytest.mark.usefixtures("_patch_get_channel_streams") - def test_streams_full_flow_prepublish_and_replay(): - """Regression test for dify#40948: `workflow_started` dropped for concurrent streaming - runs on the streams transport. - - `start_task` is invoked via `on_subscribe` after entering the subscription and publishes - both events synchronously, before the listener thread necessarily gets a chance to run its - first `xread`. `_StreamsSubscription` fixes its read boundary synchronously before spawning - that thread, so neither event is dropped. - """ - app_mode = AppMode.WORKFLOW - run_id = str(uuid.uuid4()) - - events = [{"event": "workflow_started"}, {"event": "workflow_finished"}] - - def start_task(): - _publish_events(app_mode, run_id, events) - - # MessageBasedAppGenerator is what AppGenerateService actually dispatches WORKFLOW / - # ADVANCED_CHAT streaming runs through in production. - on_subscribe = AppGenerateService._build_streaming_task_on_subscribe(start_task) - - gen = MessageBasedAppGenerator.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) - - received = [] - for msg in gen: - if isinstance(msg, str): - # skip ping events - continue - received.append(msg) - if msg.get("event") == "workflow_finished": - break - - assert [m.get("event") for m in received] == ["workflow_started", "workflow_finished"] - - @staticmethod - @pytest.mark.usefixtures("_patch_get_channel_pubsub") - def test_pubsub_full_flow_start_on_subscribe_gated(monkeypatch: pytest.MonkeyPatch): - # Speed up any potential timer if it accidentally triggers - monkeypatch.setattr("services.app_generate_service.SSE_TASK_START_FALLBACK_MS", 50) - - app_mode = AppMode.WORKFLOW - run_id = str(uuid.uuid4()) - - published_order: list[str] = [] - - def start_task(): - # When called (on subscribe), publish both events - events = [{"event": "workflow_started"}, {"event": "workflow_finished"}] - _publish_events(app_mode, run_id, events) - published_order.extend([e["event"] for e in events]) - - on_subscribe = AppGenerateService._build_streaming_task_on_subscribe(start_task) - - # Producer not started yet; only when subscribe happens - assert published_order == [] - - gen = MessageGenerator.retrieve_events(app_mode, run_id, idle_timeout=2.0, on_subscribe=on_subscribe) - - received = [] - for msg in gen: - if isinstance(msg, str): - continue - received.append(msg) - if msg.get("event") == "workflow_finished": - break - - # Verify publish happened and consumer received in order - assert published_order == ["workflow_started", "workflow_finished"] - assert [m.get("event") for m in received] == ["workflow_started", "workflow_finished"] diff --git a/api/tests/unit_tests/services/test_app_package_service.py b/api/tests/unit_tests/services/test_app_package_service.py index b4cd1e33e54a41..bfac64cddeb028 100644 --- a/api/tests/unit_tests/services/test_app_package_service.py +++ b/api/tests/unit_tests/services/test_app_package_service.py @@ -18,6 +18,8 @@ from configs import dify_config from constants.dsl_version import CURRENT_APP_DSL_VERSION +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.workflow import build_app_dsl_service from libs.archive_writer import ( ArchiveLimits, ArchiveOptions, @@ -33,7 +35,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig @@ -241,7 +242,7 @@ def _import( prepared = AppPackageService().read_package(io.BytesIO(content), accept_roster_references=preserve_agent_bindings) assert prepared is not None with prepared: - result = AppDslService(session).import_app( + result = build_app_dsl_service(session).import_app( account=account or _account(), import_mode="yaml-content", yaml_content=prepared.dsl, @@ -520,7 +521,7 @@ def mutate(manifest: AppPackageManifest, data: dict[str, object], members: dict[ result = _import(sqlite_session, _rewrite(content, mutate)) assert result.status == ImportStatus.PENDING saved_count = len(storage.saved) - confirmed = AppDslService(sqlite_session).confirm_import(import_id=result.id, account=_account()) + confirmed = build_app_dsl_service(sqlite_session).confirm_import(import_id=result.id, account=_account()) assert confirmed.status == ImportStatus.COMPLETED_WITH_WARNINGS, confirmed.error assert any( warning.code == "agent_skill_missing" and warning.path.startswith("agent_packages.agent_1.") @@ -538,9 +539,19 @@ def test_overwrite_permission_is_checked_before_resource_upload( app = _source(sqlite_session, storage) with AppPackageService().export_app(may_bundle=_bundle_any_app, app_model=app) as exported: content = exported.archive.read() - monkeypatch.setattr(AppDslService, "_load_app_for_overwrite", Mock(side_effect=NoPermissionError("Denied"))) - with pytest.raises(NoPermissionError): + target = make_app( + app_id="protected-app", + tenant_id="tenant-2", + mode=AppMode.WORKFLOW, + maintainer="another-user", + ) + sqlite_session.add(target) + sqlite_session.commit() + denied = Mock(return_value=False) + monkeypatch.setattr(AppDslService, "_check_overwrite_access", denied) + with pytest.raises(NoPermissionError, match="permission to overwrite"): _import(sqlite_session, content, app_id="protected-app") + denied.assert_called_once() assert not storage.saved @staticmethod @@ -561,7 +572,9 @@ def test_overwrite_replaces_inline_agents_and_reports_previous_owners( ) sqlite_session.commit() retirement = Mock() - monkeypatch.setattr("services.app_dsl_service.WorkflowAgentRetirementService.retire_unowned", retirement) + monkeypatch.setattr( + "services.agent.retirement_service.WorkflowAgentRetirementService.retire_unowned", retirement + ) updated = _import(sqlite_session, content, app_id=original.app_id) assert updated.status == ImportStatus.COMPLETED, updated.error assert updated.app_id == original.app_id @@ -571,8 +584,13 @@ def test_overwrite_replaces_inline_agents_and_reports_previous_owners( ) ) assert len(new_agents) == len(old_agents) == 2 + assert None not in old_agents assert not (new_agents & old_agents) - retirement.assert_called_once_with(tenant_id="tenant-2", agent_ids=old_agents, account_id="account-1") + retirement.assert_called_once_with( + tenant_id="tenant-2", + agent_ids=sorted(agent_id for agent_id in old_agents if agent_id is not None), + account_id="account-1", + ) @staticmethod @pytest.mark.parametrize("legacy_skill", [False, True]) @@ -725,7 +743,8 @@ def test_overwrite_with_mixed_graph_keeps_roster_link( sqlite_session: Session, storage: _MemoryStorage, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - "services.app_dsl_service.WorkflowAgentRetirementService.retire_unowned", lambda **_kwargs: None + "services.agent.retirement_service.WorkflowAgentRetirementService.retire_unowned", + lambda *_args, **_kwargs: None, ) content = _roster_export(sqlite_session, storage) account = _account("tenant-1") @@ -771,7 +790,7 @@ def mutate(_manifest: AppPackageManifest, data: dict[str, object], _members: dic account = _account("tenant-1") result = _import(sqlite_session, _rewrite(content, mutate), account=account, preserve_agent_bindings=True) assert result.status == ImportStatus.PENDING - confirmed = AppDslService(sqlite_session).confirm_import(import_id=result.id, account=account) + confirmed = build_app_dsl_service(sqlite_session).confirm_import(import_id=result.id, account=account) assert confirmed.status in {ImportStatus.COMPLETED, ImportStatus.COMPLETED_WITH_WARNINGS}, confirmed.error row = _bindings_by_node(sqlite_session, confirmed.app_id)["node-3"] assert row.binding_type == WorkflowAgentBindingType.ROSTER_AGENT diff --git a/api/tests/unit_tests/services/test_app_preview_details_adapters.py b/api/tests/unit_tests/services/test_app_preview_details_adapters.py index c20e3b49a740fa..ff8d7e11a1bd43 100644 --- a/api/tests/unit_tests/services/test_app_preview_details_adapters.py +++ b/api/tests/unit_tests/services/test_app_preview_details_adapters.py @@ -14,14 +14,13 @@ from sqlalchemy.orm import Session, SessionTransaction, UOWTransaction, sessionmaker from sqlalchemy.pool import QueuePool -import core.agent.tool_configuration as tool_configuration_module import core.helper.tool_parameter_cache as cache_module +import services.tools.agent_configuration as tool_configuration_module from core.agent.entities import AgentToolEntity from core.helper import encrypter from core.plugin.plugin_service import PluginService from core.tools.__base.tool import Tool from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolEntity, ToolParameter -from core.tools.tool_manager import ToolManager from extensions.ext_database import db from graphon.variables import SecretVariable from models import Account, App @@ -32,11 +31,14 @@ from models.tools import ApiToolProvider from models.workflow import Workflow, WorkflowType from repositories.app_preview_query_repository import AppPreviewQueryRepository +from repositories.tools.provider_repository import ToolProviderRepository +from repositories.tools.workflow_repository import WorkflowToolRepository from services.account_errors import AccountNotFoundError from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_preview_details_adapters import AppPreviewDetailsRuntime from services.app_preview_details_service import AppPreviewDetail from services.app_preview_query_service import AppPreviewRef, AppPreviewSiteUnavailableError, AppPreviewUnavailableError +from services.tools.tool_manager import ToolManager @dataclass(frozen=True) @@ -83,7 +85,13 @@ def assert_read_only(self) -> None: @pytest.fixture -def harness(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> Iterator[_Harness]: +def harness( + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, +) -> Iterator[_Harness]: target = App( id=str(uuid4()), tenant_id=str(uuid4()), @@ -205,7 +213,11 @@ def after_begin(_session: Session, _transaction: SessionTransaction, connection: decoy_workflow, sqlite_session_factory, sqlite_engine, - AppPreviewDetailsRuntime(details=AppPreviewQueryRepository(session_factory=factory)), + AppPreviewDetailsRuntime( + details=AppPreviewQueryRepository(session_factory=factory), + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ), closed, mutated, commits, @@ -565,7 +577,12 @@ def setex(self, key: str, expiry: int, value: str) -> None: @pytest.mark.parametrize("failure", ["none", "runtime", "decrypt"]) def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( - harness: _Harness, monkeypatch: pytest.MonkeyPatch, failure: Literal["none", "runtime", "decrypt"] + harness: _Harness, + monkeypatch: pytest.MonkeyPatch, + failure: Literal["none", "runtime", "decrypt"], + *, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, ) -> None: provider_id = str(uuid4()) agent_mode = { @@ -598,7 +615,16 @@ def test_agent_tool_masking_returns_a_copy_without_dirtying_or_flushing_models( calls: list[str] = [] checkedout_during_io: list[int] = [] - def runtime(*, tenant_id: str, app_id: str, agent_tool: AgentToolEntity, user_id: str) -> Tool: + def runtime( + *, + tenant_id: str, + app_id: str, + agent_tool: AgentToolEntity, + user_id: str, + tool_providers: ToolProviderRepository, + workflow_queries: WorkflowToolRepository, + ) -> Tool: + del workflow_queries, tool_providers assert isinstance(harness.engine.pool, QueuePool) checkedout_during_io.append(harness.engine.pool.checkedout()) harness.assert_closed() @@ -633,6 +659,8 @@ def decrypt(tenant_id: str, token: str) -> str: app_id=harness.target.id, tenant_id=harness.active_workspace.id, user_id=harness.account.id, + tool_providers=tool_providers, + workflow_queries=workflow_queries, ) assert result.model_config is not None masked_agent_mode = result.model_config["agent_mode"] diff --git a/api/tests/unit_tests/services/test_app_service.py b/api/tests/unit_tests/services/test_app_service.py index 1a5f3cbaec4f08..a758648fb11ce4 100644 --- a/api/tests/unit_tests/services/test_app_service.py +++ b/api/tests/unit_tests/services/test_app_service.py @@ -12,17 +12,19 @@ from sqlalchemy.orm import Session, sessionmaker from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from graphon.model_runtime.entities.model_entities import ModelType from models import Account, Tenant from models.account import TenantAccountJoin, TenantAccountRole from models.agent import ( Agent, AgentConfigSnapshot, + AgentHomeSnapshot, AgentIconType, AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, + AgentWorkingResourceStatus, WorkflowAgentNodeBinding, ) from models.agent_config_entities import AgentSoulConfig @@ -544,7 +546,12 @@ def record_sql(_conn, _cursor, statement, _parameters, _context, _executemany) - class TestAppResponseViewAgentConfig: def test_masking_persistent_agent_config_does_not_change_database( - self, sqlite_session: Session, sqlite_session_factory: sessionmaker[Session] + self, + sqlite_session: Session, + sqlite_session_factory: sessionmaker[Session], + *, + tool_providers, + workflow_queries, ): tenant_id = str(uuid4()) app = _persist_app(sqlite_session, tenant_id=tenant_id) @@ -565,7 +572,13 @@ def test_masking_persistent_agent_config_does_not_change_database( patch("services.app_service.mask_agent_tool_parameters", return_value=masked_agent_mode), patch.object(sqlite_session, "refresh", side_effect=AssertionError("App refresh is unnecessary")), ): - response_config = AppResponseView(app, session=sqlite_session, account=account).app_model_config + response_config = AppResponseView( + app, + session=sqlite_session, + account=account, + tool_providers=tool_providers, + workflow_queries=workflow_queries, + ).app_model_config assert response_config is not None assert response_config.agent_mode_dict == masked_agent_mode assert config.agent_mode_dict == original_agent_mode @@ -804,6 +817,10 @@ def test_update_agent_app_duplicate_name_rolls_back_and_raises_conflict(self, sq def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): app, backing_agent = _persist_agent_app(sqlite_session) + home = AgentHomeSnapshot( + id="home-1", tenant_id=app.tenant_id, agent_id=backing_agent.id, snapshot_ref="snapshot-ref" + ) + sqlite_session.add(home) workflow_app = _persist_app(sqlite_session, tenant_id=app.tenant_id, name="Workflow") workflow_app.mode = AppMode.WORKFLOW referencing_workflow = Workflow.new( @@ -860,10 +877,6 @@ def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): "services.app_service.remove_app_and_related_data_task.delay", side_effect=lambda **_kwargs: events.append("enqueue-app-cleanup"), ), - patch( - "services.app_service.AgentHomeSnapshotService.retire_all_for_agent", - return_value=["home-1"], - ) as mock_retire_homes, patch( "services.app_service.AgentWorkspaceService.retire_all_for_app", side_effect=lambda **_kwargs: events.append("retire-app-workspaces") or ["workspace-1"], @@ -906,11 +919,8 @@ def test_delete_agent_app_archives_backing_agent(self, sqlite_session: Session): tenant_id=app.tenant_id, app_id=app.id, ) - mock_retire_homes.assert_called_once_with( - session=sqlite_session, - tenant_id=app.tenant_id, - agent_id=backing_agent.id, - ) + assert home.status == AgentWorkingResourceStatus.RETIRED + assert home.retired_at is not None mock_enqueue_collection.assert_called_once_with( tenant_id=app.tenant_id, workspace_ids=["workspace-1"], diff --git a/api/tests/unit_tests/services/test_app_task_service.py b/api/tests/unit_tests/services/test_app_task_service.py index 57138d0e281e96..3b32a3b130779c 100644 --- a/api/tests/unit_tests/services/test_app_task_service.py +++ b/api/tests/unit_tests/services/test_app_task_service.py @@ -765,7 +765,7 @@ def recovery_service( ) topic = Topic(Redis(), "recovery-test") monkeypatch.setattr(topic, "publish", lambda _payload: None) - monkeypatch.setattr(task_module.MessageBasedAppGenerator, "get_response_topic", lambda *_: topic) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", lambda *_: topic) return AppTaskControlService(redis_client=recovery_redis) @staticmethod @@ -1207,7 +1207,7 @@ def publish(_channel: str, payload: bytes) -> None: topic_client = Redis() monkeypatch.setattr(topic_client, "publish", publish) topic = Topic(topic_client, "workflow-events") - monkeypatch.setattr(task_module.MessageBasedAppGenerator, "get_response_topic", lambda *_: topic) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", lambda *_: topic) assert run.task_id is not None AppTaskService.stop_workflow_task( diff --git a/api/tests/unit_tests/services/test_conversation_variable_updater.py b/api/tests/unit_tests/services/test_conversation_variable_updater.py index 6710da8b26687b..a9428cc8413aaa 100644 --- a/api/tests/unit_tests/services/test_conversation_variable_updater.py +++ b/api/tests/unit_tests/services/test_conversation_variable_updater.py @@ -3,7 +3,7 @@ from graphon.variables import StringVariable from models import ConversationVariable -from services.conversation_variable_updater import ConversationVariableUpdater +from repositories.workflow.conversation_variable_repository import WorkflowConversationVariableRepository @pytest.mark.parametrize("sqlite_session", [(ConversationVariable,)], indirect=True) @@ -24,7 +24,7 @@ def test_runtime_update_preserves_authored_id_and_other_conversations( ] sqlite_session.add_all(rows) sqlite_session.commit() - updater = ConversationVariableUpdater(sessionmaker(bind=sqlite_session.get_bind())) + updater = WorkflowConversationVariableRepository(sessionmaker(bind=sqlite_session.get_bind())) updater.update(conversation_id, variable.model_copy(update={"value": "updated"})) diff --git a/api/tests/unit_tests/services/test_github_import_urls.py b/api/tests/unit_tests/services/test_github_import_urls.py index c6cbd496fd55b0..46875ce47ff045 100644 --- a/api/tests/unit_tests/services/test_github_import_urls.py +++ b/api/tests/unit_tests/services/test_github_import_urls.py @@ -1,5 +1,7 @@ """Check URL selection at the legacy YAML import download boundary.""" +from unittest.mock import create_autospec + import httpx import pytest from sqlalchemy.orm import Session @@ -7,7 +9,9 @@ from core.file import remote_fetcher from models import Account from services.app_dsl_service import AppDslService +from services.entities.dsl_entities import AppDslOverwriteStore from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService +from services.workflow.draft_service import WorkflowDraftService @pytest.mark.parametrize("importer", ["app", "rag-pipeline"]) @@ -46,9 +50,11 @@ def fetch(method: str, url: str, **_kwargs: object) -> httpx.Response: monkeypatch.setattr(remote_fetcher, "make_request", fetch) account = Account(name="Test Account", email="test@example.com") if importer == "app": - result = AppDslService(session=unbound_session).import_app( - account=account, import_mode="yaml-url", yaml_url=url - ) + result = AppDslService( + session=unbound_session, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + overwrites=create_autospec(AppDslOverwriteStore, instance=True, spec_set=True), + ).import_app(account=account, import_mode="yaml-url", yaml_url=url) else: result = RagPipelineDslService(session=unbound_session).import_rag_pipeline( account=account, import_mode="yaml-url", yaml_url=url diff --git a/api/tests/unit_tests/services/test_human_input_delivery_test_service.py b/api/tests/unit_tests/services/test_human_input_delivery_test_service.py index 20799d2c6d8493..5f2f67d2662c56 100644 --- a/api/tests/unit_tests/services/test_human_input_delivery_test_service.py +++ b/api/tests/unit_tests/services/test_human_input_delivery_test_service.py @@ -24,14 +24,16 @@ ) from services import human_input_delivery_test_service as service_module from services.entities.feature_entities import FeatureModel -from services.human_input_delivery_test_service import ( +from services.human_input.contracts import ( DeliveryTestContext, DeliveryTestEmailRecipient, DeliveryTestError, - DeliveryTestRegistry, DeliveryTestResult, DeliveryTestStatus, DeliveryTestUnsupportedError, +) +from services.human_input_delivery_test_service import ( + DeliveryTestRegistry, EmailDeliveryTestHandler, HumanInputDeliveryTestService, _build_form_link, diff --git a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py index d236ed42b397fc..687f9a726d1c5f 100644 --- a/api/tests/unit_tests/services/test_installed_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_installed_app_generation_adapters.py @@ -12,10 +12,10 @@ from sqlalchemy.orm import Session, sessionmaker import services.app_generate_service as generation_module -from core.app.apps import message_based_app_generator from core.app.entities.app_invoke_entities import InvokeFrom from core.app.features.rate_limiting.rate_limit import RateLimit, RateLimitGenerator from enums import DeploymentEdition +from extensions.application_services.workflow import WorkflowExecutionDependencies from extensions.ext_database import db from extensions.ext_redis import RedisClientWrapper from libs.broadcast_channel.redis.streams_channel import ( @@ -27,10 +27,13 @@ from models.enums import ConversationFromSource from models.workflow import WorkflowType from services.account_errors import AccountNotFoundError +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.errors.conversation import ConversationNotExistsError from services.installed_app_generation_adapters import AppGenerateServiceRuntime from services.installed_app_generation_service import GenerationResponse +from services.workflow.execution.adapters import response_stream +from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} _WORKFLOW_ARGS: dict[str, object] = {"inputs": {"count": 0}, "files": []} @@ -47,7 +50,12 @@ class _RuntimeHarness: @pytest.fixture -def harness(sqlite_session_factory: sessionmaker[Session]) -> _RuntimeHarness: +def harness( + sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowExecutionDependencies, +) -> _RuntimeHarness: with sqlite_session_factory.begin() as session: app = App( tenant_id=str(uuid4()), name="Original app", mode=AppMode.COMPLETION, enable_site=True, enable_api=True @@ -76,7 +84,9 @@ def record_commit(session: Session) -> None: committed_sessions.append(session) return _RuntimeHarness( - AppGenerateServiceRuntime(session_factory=cast(sessionmaker[Session], factory)), + AppGenerateServiceRuntime( + session_factory=cast(sessionmaker[Session], factory), variables=workflow_variables, runtime=workflow_runtime + ), app.id, account.id, closed_sessions, @@ -113,8 +123,10 @@ def assert_context( assert streaming is expected_streaming return generate(session) - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, @@ -122,13 +134,15 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse | Generator[Mapping[str, object] | str]: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert args == _ARGS return assert_context( session=session, app_model=app_model, user=user, invoke_from=invoke_from, streaming=streaming ) expected_streaming = streaming - monkeypatch.setattr(generation_module.AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(generation_module.AppGenerateService, "generate", boundary_generate) def _invoke_generation( @@ -567,7 +581,7 @@ def test_workflow_dispatch_starts_task_after_subscription_with_runtime_session_c close = mocker.spy(_StreamsSubscription, "close") def activate_subscription(subscription: _StreamsSubscription) -> None: - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) if not transport_events: transport_events.append("subscribe") @@ -577,12 +591,12 @@ def activate_subscription(subscription: _StreamsSubscription) -> None: start_listener = mocker.patch.object( _StreamsSubscription, "_start_if_needed", autospec=True, side_effect=activate_subscription ) - monkeypatch.setattr(message_based_app_generator, "get_pubsub_broadcast_channel", lambda: channel) + monkeypatch.setattr(response_stream, "get_pubsub_broadcast_channel", lambda: channel) submitted: list[generation_module.AppExecutionParams] = [] def enqueue(payload_json: str) -> None: assert transport_events == ["subscribe"] - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) submitted.append(generation_module.AppExecutionParams.model_validate_json(payload_json)) transport_events.append("enqueue") @@ -592,7 +606,7 @@ def enqueue(payload_json: str) -> None: result = _generate_workflow(harness, sqlite_session_factory, args) assert isinstance(result, RateLimitGenerator) - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert harness.committed_sessions == [harness.closed_sessions[1]] assert submitted == [] prepare.assert_called_once() @@ -652,7 +666,7 @@ def test_unpublished_workflow_raises_before_subscription_or_task_creation( redis, commands = redis_transport channel = StreamsBroadcastChannel(redis._require_client()) topic = mocker.spy(channel, "topic") - monkeypatch.setattr(message_based_app_generator, "get_pubsub_broadcast_channel", lambda: channel) + monkeypatch.setattr(response_stream, "get_pubsub_broadcast_channel", lambda: channel) enqueue = MagicMock() monkeypatch.setattr(generation_module.workflow_based_app_execution_task, "delay", enqueue) @@ -660,7 +674,7 @@ def test_unpublished_workflow_raises_before_subscription_or_task_creation( _generate_workflow(harness, sqlite_session_factory, _WORKFLOW_ARGS) assert type(raised.value) is ValueError - assert len(harness.closed_sessions) == 2 + assert len(set(harness.closed_sessions)) == 2 assert all(not session.in_transaction() for session in harness.closed_sessions) assert harness.committed_sessions == [] topic.assert_not_called() diff --git a/api/tests/unit_tests/services/test_skill_management_service.py b/api/tests/unit_tests/services/test_skill_management_service.py index 49dc21e6e678f1..3f54a7fb30f8d8 100644 --- a/api/tests/unit_tests/services/test_skill_management_service.py +++ b/api/tests/unit_tests/services/test_skill_management_service.py @@ -19,6 +19,7 @@ from core.credit_usage import CreditUsageCreatedBy from core.db.session_factory import session_factory from core.tools.tool_file_manager import ToolFileManager +from enums.agent import WorkflowAgentBindingType from models.account import Account from models.agent import ( Agent, @@ -31,7 +32,6 @@ AgentScope, AgentSource, AgentStatus, - WorkflowAgentBindingType, WorkflowAgentNodeBinding, ) from models.agent_config_entities import ( diff --git a/api/tests/unit_tests/services/test_snippet_dsl_service.py b/api/tests/unit_tests/services/test_snippet_dsl_service.py index 3d203cf08e9976..0519ca3b475173 100644 --- a/api/tests/unit_tests/services/test_snippet_dsl_service.py +++ b/api/tests/unit_tests/services/test_snippet_dsl_service.py @@ -10,6 +10,7 @@ from sqlalchemy import event, select from sqlalchemy.orm import Session +from extensions.application_services.snippets import build_snippet_dsl_service, build_snippet_service from graphon.nodes import BuiltinNodeTypes from models import Account from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding @@ -24,7 +25,6 @@ SnippetPendingData, _check_version_compatibility, ) -from services.snippet_service import SnippetService from tests.helpers.unit.model_factories import make_account, make_tenant, make_workflow SQLITE_MODELS = (CustomizedSnippet,) @@ -39,7 +39,7 @@ @pytest.fixture def service(sqlite_session: Session) -> SnippetDslService: """Create the service with a real caller-owned SQLite session.""" - return SnippetDslService(session=sqlite_session) + return build_snippet_dsl_service(session=sqlite_session) @pytest.fixture @@ -553,7 +553,7 @@ def forbidden_yaml_load(*_args: object, **_kwargs: object) -> NoReturn: assert snippet.created_by == account.id assert snippet.name == "Override name" assert snippet.description == "Override description" - assert SnippetService(session=sqlite_session).get_draft_workflow(snippet) is not None + assert build_snippet_service(session=sqlite_session).get_draft_workflow(snippet) is not None assert commands[-1] == ("DEL", redis_key) assert redis_key not in values @@ -683,10 +683,16 @@ def test_create_or_update_snippet_updates_existing_snippet_and_syncs_workflow( retirement_calls: list[dict[str, object]] = [] original_retire = WorkflowAgentRetirementService.retire_unowned - def retire_unowned(*, tenant_id: str, agent_ids: Iterable[str], account_id: str | None) -> None: + def retire_unowned( + retirement: WorkflowAgentRetirementService, + *, + tenant_id: str, + agent_ids: Iterable[str], + account_id: str | None, + ) -> None: assert not sqlite_session.in_transaction() retirement_calls.append({"tenant_id": tenant_id, "agent_ids": set(agent_ids), "account_id": account_id}) - original_retire(tenant_id=tenant_id, agent_ids=agent_ids, account_id=account_id) + original_retire(retirement, tenant_id=tenant_id, agent_ids=agent_ids, account_id=account_id) monkeypatch.setattr(WorkflowAgentRetirementService, "retire_unowned", retire_unowned) @@ -710,10 +716,14 @@ def retire_unowned(*, tenant_id: str, agent_ids: Iterable[str], account_id: str assert snippet.type == "node" assert snippet.icon_info == {"icon": "x"} assert not sqlite_session.in_transaction() + persisted_draft = sqlite_session.scalar(select(Workflow).where(Workflow.app_id == result.id)) + assert persisted_draft is not None + assert persisted_draft.kind == "snippet" + assert persisted_draft.graph_dict == {"nodes": [], "edges": []} persisted = sqlite_session.get(CustomizedSnippet, snippet.id) assert persisted is not None assert persisted.name == "New" - workflow = SnippetService(session=sqlite_session).get_draft_workflow(snippet) + workflow = build_snippet_service(session=sqlite_session).get_draft_workflow(snippet) assert workflow is draft_workflow assert workflow.graph_dict == {"nodes": [], "edges": []} assert sqlite_session.get(WorkflowAgentNodeBinding, "retired-binding") is None @@ -745,7 +755,7 @@ def test_create_or_update_snippet_creates_new_snippet_and_flushes(service: Snipp assert result.type == "group" assert not sqlite_session.in_transaction() assert sqlite_session.get(CustomizedSnippet, result.id) is result - workflow = SnippetService(session=sqlite_session).get_draft_workflow(result) + workflow = build_snippet_service(session=sqlite_session).get_draft_workflow(result) assert workflow is not None assert workflow.graph_dict == {"nodes": [], "edges": []} assert workflow.created_by == "account-1" diff --git a/api/tests/unit_tests/services/test_snippet_generate_service.py b/api/tests/unit_tests/services/test_snippet_generate_service.py index de10514933366f..008048c04a9404 100644 --- a/api/tests/unit_tests/services/test_snippet_generate_service.py +++ b/api/tests/unit_tests/services/test_snippet_generate_service.py @@ -1,14 +1,12 @@ import json -import unittest.mock from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock, create_autospec import pytest from sqlalchemy.orm import Session, sessionmaker -from controllers.console.app.workflow import LoopNodeRunPayload from core.app.app_config.features.file_upload.manager import FileUploadConfigManager -from core.app.apps.workflow.app_generator import WorkflowAppGenerator -from core.workflow.node_factory import DifyNodeFactory from core.workflow.snippet_start import SNIPPET_VIRTUAL_START_NODE_ID from graphon.file import FileUploadConfig from graphon.file.models import File @@ -17,13 +15,46 @@ from models.account import Account from models.snippet import CustomizedSnippet, SnippetType from models.workflow import Workflow, WorkflowKind, WorkflowNodeExecutionModel +from repositories.workflow.definition_repository import workflow_snapshot from services.snippet_generate_service import SnippetGenerateService -from services.snippet_service import SnippetService +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.ports import WorkflowRuntime +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_service import WorkflowService from tests.helpers.unit.model_factories import make_account, make_workflow from tests.workflow_test_utils import build_test_graph_init_params +class SnippetReader: + workflow: Workflow | None = None + + def get_draft_workflow(self, snippet: CustomizedSnippet) -> Workflow | None: + assert snippet.id == "snippet-1" + return self.workflow + + +@pytest.fixture +def snippet_reader() -> SnippetReader: + return SnippetReader() + + +@pytest.fixture +def generation( + snippet_reader: SnippetReader, + workflow_variables: WorkflowExecutionVariables, + sqlite_session_factory: sessionmaker[Session], + *, + workflow_runtime: WorkflowRuntime, +) -> SnippetGenerateService: + return SnippetGenerateService( + snippets=snippet_reader, + variables=workflow_variables, + workflows=WorkflowService(sqlite_session_factory), + runtime=workflow_runtime, + ) + + def _workflow(graph: dict) -> Workflow: return make_workflow(workflow_id="workflow-1", app_id="snippet-1", kind=WorkflowKind.SNIPPET, graph=graph) @@ -44,22 +75,45 @@ def _account(account_id: str = "user-1") -> Account: return make_account(account_id=account_id, name="Test User", email=f"{account_id}@example.com") -def _set_draft_workflow( +@pytest.mark.parametrize("entry", ["generate", "iteration", "loop"]) +def test_debug_generators_use_injected_variable_dependencies( + generation: SnippetGenerateService, + snippet_reader: SnippetReader, + workflow_variables: WorkflowExecutionVariables, monkeypatch: pytest.MonkeyPatch, - workflow: Workflow | None, + entry: str, ) -> None: - def get_draft_workflow(_self, *, snippet): - _ = snippet - return workflow + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import BuiltinNodeTypes + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator - monkeypatch.setattr(SnippetService, "get_draft_workflow", get_draft_workflow) - - -def _set_published_workflow( - monkeypatch: pytest.MonkeyPatch, - workflow: Workflow | None, -) -> None: - monkeypatch.setattr(SnippetService, "get_published_workflow", lambda _self, _snippet: workflow) + workflow = _workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []}) + snippet_reader.workflow = workflow + user = _account() + calls: list[str] = [] + + def execute(generator: WorkflowAppGenerator, **_kwargs: object) -> dict[str, str]: + assert generator.draft_variable_loader == workflow_variables.workflow_loader + assert generator.draft_variable_saver == workflow_variables.saver_factory + saver = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, user, tenant_id=workflow.tenant_id) + saver(workflow.app_id, "node", BuiltinNodeTypes.CODE, "execution").save( + process_data=None, outputs={"text": "injected"} + ) + loader = workflow_variables.workflow_loader(workflow_snapshot(workflow), user.id) + assert loader.load_variables([["node", "text"]])[0].value == "injected" + calls.append(entry) + return {"result": "ready"} + + method = {"generate": "generate", "iteration": "single_iteration_generate", "loop": "single_loop_generate"}[entry] + monkeypatch.setattr(WorkflowAppGenerator, method, execute) + if entry == "generate": + result = generation.generate(_snippet(), user, {}, InvokeFrom.DEBUGGER, streaming=False) + elif entry == "iteration": + result = generation.generate_single_iteration(_snippet(), user, "node", {}, streaming=False) + else: + result = generation.generate_single_loop(_snippet(), user, "node", {}, streaming=False) + assert result == {"result": "ready"} + assert calls == [entry] def test_filter_virtual_start_events_keeps_blocking_response_unchanged(): @@ -96,7 +150,7 @@ def test_is_virtual_start_event(message, expected): def test_ensure_start_node_returns_workflow_when_start_already_exists(): - workflow = _workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []}) + workflow = workflow_snapshot(_workflow({"nodes": [{"id": "start", "data": {"type": "start"}}], "edges": []})) snippet = _snippet() result = SnippetGenerateService._ensure_start_node(workflow, snippet) @@ -104,7 +158,7 @@ def test_ensure_start_node_returns_workflow_when_start_already_exists(): assert result is workflow -def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch: pytest.MonkeyPatch): +def test_ensure_start_node_injects_virtual_start_for_root_candidates(): graph = { "nodes": [ {"id": "llm-1", "data": {"type": "llm"}}, @@ -112,7 +166,7 @@ def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch ], "edges": [{"source": "llm-1", "target": "answer-1"}], } - workflow = _workflow(graph) + workflow = workflow_snapshot(_workflow(graph)) snippet = _snippet( input_fields=[ { @@ -124,18 +178,15 @@ def test_ensure_start_node_injects_virtual_start_for_root_candidates(monkeypatch } ] ) - make_transient = unittest.mock.Mock() - monkeypatch.setattr("services.snippet_generate_service.make_transient", make_transient) - result = SnippetGenerateService._ensure_start_node(workflow, snippet) - assert result is workflow - updated_graph = workflow.graph_dict + assert result is not workflow + assert workflow.graph_dict == graph + updated_graph = result.graph_dict assert updated_graph["nodes"][0]["id"] == SNIPPET_VIRTUAL_START_NODE_ID assert updated_graph["nodes"][0]["data"]["variables"][0]["max_length"] == 128 assert updated_graph["edges"][-1]["source"] == SNIPPET_VIRTUAL_START_NODE_ID assert updated_graph["edges"][-1]["target"] == "llm-1" - make_transient.assert_called_once_with(workflow) def _nested_snippet_graph(ownership: str, outer_type: str) -> dict: @@ -298,24 +349,24 @@ def build_from_mappings(**kwargs): def test_generate_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - _set_draft_workflow(monkeypatch, None) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate( + generation.generate( snippet=_snippet(), user=_account(), args={"inputs": {}}, invoke_from="debugger", - session_maker=unbound_session_factory, ) def test_generate_delegates_to_workflow_generator_and_filters_stream( monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) snippet = _snippet() @@ -326,88 +377,39 @@ def test_generate_delegates_to_workflow_generator_and_filters_stream( {"event": "node_finished", "data": {"node_id": "llm-1"}}, ] ) - generate_kwargs: dict[str, object] = {} - ensure_calls: list[tuple[Workflow, CustomizedSnippet]] = [] - converted: list[object] = [] - - def generate(_self, **kwargs): - generate_kwargs.update(kwargs) - return raw_stream - - def ensure_start_node(_cls, current_workflow, current_snippet): - ensure_calls.append((current_workflow, current_snippet)) - return current_workflow - - def convert_to_event_stream(response): - converted.append(response) - return response - - _set_draft_workflow(monkeypatch, workflow) - monkeypatch.setattr(WorkflowAppGenerator, "generate", generate) - monkeypatch.setattr(WorkflowAppGenerator, "convert_to_event_stream", staticmethod(convert_to_event_stream)) - monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", classmethod(ensure_start_node)) - - result = SnippetGenerateService.generate( + generator = SimpleNamespace(generate=Mock(return_value=raw_stream)) + workflow_generator_class = Mock(return_value=generator) + snippet_reader.workflow = workflow + expected_workflow = workflow_snapshot(workflow) + ensure_start_node = Mock(return_value=expected_workflow) + monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", ensure_start_node) + monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) + + result = generation.generate( snippet=snippet, user=user, args={"inputs": {"query": "hello"}}, invoke_from="debugger", - session_maker=unbound_session_factory, ) - assert list(result) == [{"event": "node_finished", "data": {"node_id": "llm-1"}}] - assert ensure_calls == [(workflow, snippet)] - kwargs = generate_kwargs + assert [json.loads(event.removeprefix("data: ")) for event in result] == [ + {"event": "node_finished", "data": {"node_id": "llm-1"}} + ] + ensure_start_node.assert_called_once_with(expected_workflow, snippet) + generator.generate.assert_called_once() + kwargs = generator.generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == expected_workflow assert kwargs["user"] is user assert kwargs["streaming"] is True assert kwargs["call_depth"] == 0 - assert len(converted) == 1 - - -def test_run_published_delegates_to_workflow_generator_non_streaming( - monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], -): - workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) - snippet = _snippet() - user = _account() - generate_kwargs: dict[str, object] = {} - ensure_calls: list[tuple[Workflow, CustomizedSnippet]] = [] - - def generate(_self, **kwargs): - generate_kwargs.update(kwargs) - return {"data": {"outputs": {"answer": "ok"}}} - - def ensure_start_node(_cls, current_workflow, current_snippet): - ensure_calls.append((current_workflow, current_snippet)) - return current_workflow - - _set_published_workflow(monkeypatch, workflow) - monkeypatch.setattr(WorkflowAppGenerator, "generate", generate) - monkeypatch.setattr(SnippetGenerateService, "_ensure_start_node", classmethod(ensure_start_node)) - - result = SnippetGenerateService.run_published( - snippet=snippet, - user=user, - args={"inputs": {"query": "hello"}}, - invoke_from="service-api", - session_maker=unbound_session_factory, - ) - - assert result == {"data": {"outputs": {"answer": "ok"}}} - assert ensure_calls == [(workflow, snippet)] - kwargs = generate_kwargs - assert kwargs["app_model"].id == "snippet-1" - assert kwargs["streaming"] is False - assert kwargs["call_depth"] == 0 def test_ensure_start_node_for_worker_delegates(monkeypatch: pytest.MonkeyPatch): workflow = _workflow({"nodes": [], "edges": []}) snippet = _snippet() - ensure_calls: list[tuple[Workflow, CustomizedSnippet]] = [] + ensure_calls: list[tuple[WorkflowSnapshot, CustomizedSnippet]] = [] def ensure_start_node(_cls, current_workflow, current_snippet): ensure_calls.append((current_workflow, current_snippet)) @@ -418,190 +420,155 @@ def ensure_start_node(_cls, current_workflow, current_snippet): result = SnippetGenerateService.ensure_start_node_for_worker(workflow, snippet) assert result is workflow - assert ensure_calls == [(workflow, snippet)] + assert ensure_calls == [(workflow_snapshot(workflow), snippet)] def test_run_draft_node_delegates_to_workflow_service( - monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): workflow = _workflow({"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}) snippet = _snippet() account = _account("account-1") execution = WorkflowNodeExecutionModel(id="execution-1") - run_kwargs: dict[str, object] = {} - - def run_draft_workflow_node(_self, **kwargs): - run_kwargs.update(kwargs) - return execution + workflow_service = create_autospec(WorkflowService, instance=True, spec_set=True) + workflow_service.run_draft_workflow_node.return_value = execution - workflow_service = WorkflowService(unbound_session_factory) - _set_draft_workflow(monkeypatch, workflow) - monkeypatch.setattr(WorkflowService, "run_draft_workflow_node", run_draft_workflow_node) - monkeypatch.setattr("services.snippet_generate_service.WorkflowService", lambda: workflow_service) + snippet_reader.workflow = workflow + generation._workflows = workflow_service - result = SnippetGenerateService.run_draft_node( + result = generation.run_draft_node( snippet=snippet, node_id="llm-1", user_inputs={"query": "hello"}, account=account, query="question", files=[], - session_maker=unbound_session_factory, ) assert result is execution - kwargs = run_kwargs + workflow_service.run_draft_workflow_node.assert_called_once() + kwargs = workflow_service.run_draft_workflow_node.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["draft_workflow"] is workflow + assert isinstance(kwargs["draft_workflow"], WorkflowSnapshot) + assert kwargs["draft_workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "llm-1" assert kwargs["user_inputs"] == {"query": "hello"} assert kwargs["account"] is account assert kwargs["query"] == "question" assert kwargs["files"] == [] + assert kwargs["variables"] is workflow_variables def test_run_draft_node_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], + *, + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - _set_draft_workflow(monkeypatch, None) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.run_draft_node( + generation.run_draft_node( snippet=_snippet(), node_id="llm-1", user_inputs={}, account=_account("account-1"), - session_maker=unbound_session_factory, ) def test_generate_single_iteration_delegates_to_workflow_generator( monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ) -> None: workflow = _workflow({"nodes": [{"id": "iteration-1", "data": {"type": "iteration"}}], "edges": []}) snippet = _snippet() user = _account() response = iter(["event"]) - generate_kwargs: dict[str, object] = {} - converted: list[object] = [] - - def single_iteration_generate(_self, **kwargs): - generate_kwargs.update(kwargs) - return response - - def convert_to_event_stream(item): - converted.append(item) - return item - - _set_draft_workflow(monkeypatch, workflow) - monkeypatch.setattr(WorkflowAppGenerator, "single_iteration_generate", single_iteration_generate) - monkeypatch.setattr(WorkflowAppGenerator, "convert_to_event_stream", staticmethod(convert_to_event_stream)) + generator = SimpleNamespace(single_iteration_generate=Mock(return_value=response)) + workflow_generator_class = Mock(return_value=generator) + snippet_reader.workflow = workflow + monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) - result = SnippetGenerateService.generate_single_iteration( + result = generation.generate_single_iteration( snippet=snippet, user=user, node_id="iteration-1", args={"inputs": {"items": [1]}}, - session_maker=sqlite_session_factory, ) - assert list(result) == ["event"] - kwargs = generate_kwargs + assert list(result) == ["event: event\n\n"] + generator.single_iteration_generate.assert_called_once() + kwargs = generator.single_iteration_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "iteration-1" assert kwargs["user"] is user assert kwargs["streaming"] is True - assert isinstance(kwargs["session"], Session) - assert converted == [response] + assert "session" not in kwargs def test_generate_single_iteration_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, unbound_session_factory: sessionmaker[Session] + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - _set_draft_workflow(monkeypatch, None) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate_single_iteration( + generation.generate_single_iteration( snippet=_snippet(), user=_account(), node_id="iteration-1", args={"inputs": {}}, - session_maker=unbound_session_factory, ) def test_generate_single_loop_delegates_to_workflow_generator( monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ) -> None: workflow = _workflow({"nodes": [{"id": "loop-1", "data": {"type": "loop"}}], "edges": []}) snippet = _snippet() user = _account() response = iter(["event"]) - generate_kwargs: dict[str, object] = {} - converted: list[object] = [] - - def single_loop_generate(_self, **kwargs): - generate_kwargs.update(kwargs) - return response - - def convert_to_event_stream(item): - converted.append(item) - return item - - _set_draft_workflow(monkeypatch, workflow) - monkeypatch.setattr(WorkflowAppGenerator, "single_loop_generate", single_loop_generate) - monkeypatch.setattr(WorkflowAppGenerator, "convert_to_event_stream", staticmethod(convert_to_event_stream)) + generator = SimpleNamespace(single_loop_generate=Mock(return_value=response)) + workflow_generator_class = Mock(return_value=generator) + snippet_reader.workflow = workflow + monkeypatch.setattr("services.snippet_generate_service.WorkflowAppGenerator", workflow_generator_class) - result = SnippetGenerateService.generate_single_loop( + result = generation.generate_single_loop( snippet=snippet, user=user, node_id="loop-1", - args=LoopNodeRunPayload(inputs={"items": [1]}), - session_maker=sqlite_session_factory, + args={"inputs": {"items": [1]}}, ) - assert list(result) == ["event"] - kwargs = generate_kwargs + assert list(result) == ["event: event\n\n"] + generator.single_loop_generate.assert_called_once() + kwargs = generator.single_loop_generate.call_args.kwargs assert kwargs["app_model"].id == "snippet-1" - assert kwargs["workflow"] is workflow + assert isinstance(kwargs["workflow"], WorkflowSnapshot) + assert kwargs["workflow"] == workflow_snapshot(workflow) assert kwargs["node_id"] == "loop-1" assert kwargs["user"] is user assert kwargs["streaming"] is True - assert isinstance(kwargs["session"], Session) - assert converted == [response] + assert "session" not in kwargs def test_generate_single_loop_raises_when_draft_workflow_missing( - monkeypatch: pytest.MonkeyPatch, unbound_session_factory: sessionmaker[Session] + generation: SnippetGenerateService, + snippet_reader: SnippetReader, ): - _set_draft_workflow(monkeypatch, None) + snippet_reader.workflow = None with pytest.raises(ValueError, match="Workflow not initialized"): - SnippetGenerateService.generate_single_loop( + generation.generate_single_loop( snippet=_snippet(), user=_account(), node_id="loop-1", - args=LoopNodeRunPayload(inputs={}), - session_maker=unbound_session_factory, - ) - - -def test_run_published_raises_when_published_workflow_missing( - monkeypatch: pytest.MonkeyPatch, - unbound_session_factory: sessionmaker[Session], -): - _set_published_workflow(monkeypatch, None) - - with pytest.raises(ValueError, match="No published workflow found"): - SnippetGenerateService.run_published( - snippet=_snippet(), - user=_account(), args={"inputs": {}}, - invoke_from="service-api", - session_maker=unbound_session_factory, ) diff --git a/api/tests/unit_tests/services/test_snippet_service.py b/api/tests/unit_tests/services/test_snippet_service.py index 605db61c2b3b6d..678e7523c4f863 100644 --- a/api/tests/unit_tests/services/test_snippet_service.py +++ b/api/tests/unit_tests/services/test_snippet_service.py @@ -3,25 +3,20 @@ import json from datetime import datetime from types import SimpleNamespace -from unittest.mock import Mock +from unittest.mock import ANY, Mock import pytest from sqlalchemy import event, select from sqlalchemy.orm import Session, sessionmaker from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType +from extensions.application_services.snippets import build_snippet_service from extensions.storage.storage_type import StorageType from graphon.variables.segments import StringSegment from graphon.variables.types import SegmentType from models.account import Account -from models.agent import ( - Agent, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.enums import AppStatus, CreatorUserRole from models.model import App, AppMode, UploadFile from models.snippet import CustomizedSnippet, SnippetType @@ -34,10 +29,10 @@ WorkflowRun, WorkflowType, ) -from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError +from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError from services.snippet_service import SnippetService from services.tag_application_service import TagApplicationService +from services.workflow.snippet_policy import validate_snippet_graph_forbidden_nodes from tests.helpers.unit.model_factories import make_account @@ -83,7 +78,7 @@ def test_create_snippet_allows_duplicate_names( existing.name = "shared name" sqlite_session.add(existing) sqlite_session.commit() - service = SnippetService(session_maker=sqlite_session_factory) + service = build_snippet_service(session_maker=sqlite_session_factory) snippet = service.create_snippet( tenant_id="tenant-1", @@ -105,7 +100,7 @@ def test_create_snippet_allows_duplicate_names( def test_validate_snippet_graph_forbidden_nodes_ignores_malformed_nodes() -> None: - SnippetService.validate_snippet_graph_forbidden_nodes( + validate_snippet_graph_forbidden_nodes( { "nodes": [ "not-a-node", @@ -119,7 +114,7 @@ def test_validate_snippet_graph_forbidden_nodes_ignores_malformed_nodes() -> Non def test_validate_snippet_graph_forbidden_nodes_raises_with_node_details() -> None: with pytest.raises(ValueError, match="start-1:start"): - SnippetService.validate_snippet_graph_forbidden_nodes({"nodes": [{"id": "start-1", "data": {"type": "start"}}]}) + validate_snippet_graph_forbidden_nodes({"nodes": [{"id": "start-1", "data": {"type": "start"}}]}) def test_get_snippets_returns_empty_when_tag_filter_has_no_targets( @@ -217,7 +212,7 @@ def test_get_snippets_keyword_treats_like_wildcards_literally( ) ) sqlite_session.flush() - service = SnippetService(session=sqlite_session) + service = build_snippet_service(session=sqlite_session) result, total, has_more = service.get_snippets( tenant_id="tenant-1", session=sqlite_session, keyword=keyword, tags=application_tags @@ -272,105 +267,6 @@ def test_update_snippet_updates_optional_fields(sqlite_session: Session) -> None assert stored.description == "new description" -def test_sync_draft_workflow_creates_draft_and_updates_input_fields( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=None)) - monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", - Mock(return_value={"retired-agent"}), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.snippet_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - snippet = _snippet() - account = _account() - - workflow = service.sync_draft_workflow( - snippet=snippet, - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - unique_hash=None, - account=account, - input_fields=[{"variable": "query"}], - ) - - assert workflow.app_id == snippet.id - assert workflow.kind == WorkflowKind.SNIPPET - assert json.loads(snippet.input_fields) == [{"variable": "query"}] - sqlite_session.expire_all() - stored_workflow = sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) - stored_snippet = sqlite_session.get(CustomizedSnippet, snippet.id) - assert stored_workflow is not None - assert stored_snippet is not None - assert stored_snippet.input_fields_list == [{"variable": "query"}] - retire_unowned.assert_called_once_with( - tenant_id=snippet.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - -def test_sync_draft_workflow_raises_when_hash_mismatches( - sqlite_session_factory: sessionmaker[Session], -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - draft_workflow = _create_workflow( - workflow_id="workflow-1", version=Workflow.VERSION_DRAFT, graph={"nodes": []}, features={} - ) - service.get_draft_workflow = Mock(return_value=draft_workflow) - - with pytest.raises(WorkflowHashNotEqualError): - service.sync_draft_workflow( - snippet=_snippet(), - graph={"nodes": [], "edges": []}, - unique_hash="client-hash", - account=_account(), - ) - - -def test_sync_draft_workflow_updates_existing_draft_and_clears_variables( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - service = SnippetService(session_maker=sqlite_session_factory) - workflow = _create_workflow( - workflow_id="workflow-1", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - unique_hash = workflow.unique_hash - snippet = _snippet() - account = _account() - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=workflow)) - - result = service.sync_draft_workflow( - snippet=snippet, - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - unique_hash=unique_hash, - account=account, - input_fields=[{"variable": "query"}], - ) - - assert result is workflow - assert workflow.graph_dict["nodes"][0]["id"] == "llm-1" - assert workflow.type == WorkflowType.WORKFLOW - assert workflow.kind == WorkflowKind.SNIPPET - assert workflow.updated_by == account.id - assert workflow.environment_variables == [] - assert workflow.conversation_variables == [] - assert json.loads(snippet.input_fields) == [{"variable": "query"}] - sqlite_session.expire_all() - assert sqlite_session.get(Workflow, workflow.id) is not None - assert sqlite_session.get(CustomizedSnippet, snippet.id) is not None - - def test_update_workflow_updates_marked_fields(sqlite_session: Session) -> None: service = SnippetService.__new__(SnippetService) workflow = _create_workflow( @@ -392,7 +288,9 @@ def test_update_workflow_updates_marked_fields(sqlite_session: Session) -> None: data={"marked_name": "v1", "marked_comment": "first version", "ignored": "value"}, ) - assert result is workflow + assert result is not None + assert result.id == workflow.id + assert result.marked_name == "v1" assert workflow.marked_name == "v1" assert workflow.marked_comment == "first version" assert workflow.updated_by == "account-1" @@ -416,62 +314,34 @@ def test_update_workflow_returns_none_when_missing(sqlite_session: Session) -> N assert result is None -def test_delete_workflow_removes_published_version(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version="2026-01-01 00:00:00", - graph={"nodes": []}, - features={}, - ) - snippet = _snippet(workflow_id="workflow-2") - sqlite_session.add_all([snippet, workflow]) - sqlite_session.flush() - - result = service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") - - assert result is True - sqlite_session.flush() - assert sqlite_session.get(Workflow, "workflow-1") is None - - -def test_delete_workflow_raises_when_missing(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - - with pytest.raises(ValueError, match="not found"): - service.delete_workflow(session=sqlite_session, snippet=_snippet(), workflow_id="missing-workflow") - - -def test_delete_workflow_raises_for_draft_version(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version=Workflow.VERSION_DRAFT, - graph={"nodes": []}, - features={}, +@pytest.mark.parametrize("mismatch", ["tenant_id", "app_id", "kind", "version"]) +def test_update_workflow_preserves_snippet_owner_kind_and_published_constraints( + sqlite_session: Session, mismatch: str +) -> None: + workflow = _create_workflow(workflow_id="workflow-1", version="published", graph={"nodes": []}, features={}) + setattr( + workflow, + mismatch, + { + "tenant_id": "another-tenant", + "app_id": "another-snippet", + "kind": WorkflowKind.STANDARD.value, + "version": Workflow.VERSION_DRAFT, + }[mismatch], ) - snippet = _snippet() - sqlite_session.add_all([snippet, workflow]) + sqlite_session.add(workflow) sqlite_session.flush() + before = (workflow.marked_name, workflow.marked_comment, workflow.updated_by, workflow.updated_at) - with pytest.raises(DraftWorkflowDeletionError): - service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") - - -def test_delete_workflow_raises_when_currently_active(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) - workflow = _create_workflow( - workflow_id="workflow-1", - version="2026-01-01 00:00:00", - graph={"nodes": []}, - features={}, + result = SnippetService.__new__(SnippetService).update_workflow( + session=sqlite_session, + snippet=_snippet(), + workflow_id=workflow.id, + account=_account(), + data={"marked_name": "changed", "marked_comment": "changed"}, ) - snippet = _snippet(workflow_id="workflow-1") - sqlite_session.add_all([snippet, workflow]) - sqlite_session.flush() - - with pytest.raises(WorkflowInUseError): - service.delete_workflow(session=sqlite_session, snippet=snippet, workflow_id="workflow-1") + assert result is None + assert (workflow.marked_name, workflow.marked_comment, workflow.updated_by, workflow.updated_at) == before def test_get_default_block_configs_skips_empty_defaults(monkeypatch: pytest.MonkeyPatch) -> None: @@ -516,119 +386,6 @@ def test_get_default_block_config_returns_none_for_empty_default(monkeypatch: py assert service.get_default_block_config("llm") is None -def test_restore_published_snippet_workflow_to_draft_copies_source_snapshot( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - snippet = _snippet() - account = _account("account-2") - source_graph = {"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []} - source_features = {"opening_statement": "hello"} - source_workflow = _create_workflow( - workflow_id="published-workflow", - version="2026-04-28 00:00:00", - graph=source_graph, - features=source_features, - ) - draft_workflow = _create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=source_workflow)) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=draft_workflow)) - monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.restore_agent_node_bindings_to_draft", - Mock(return_value={"retired-agent"}), - ) - retire_unowned = Mock() - monkeypatch.setattr( - "services.snippet_service.WorkflowAgentRetirementService.retire_unowned", - retire_unowned, - ) - - result = service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=source_workflow.id, - account=account, - ) - - assert result is draft_workflow - assert draft_workflow.graph_dict == source_graph - assert draft_workflow.features_dict == source_features - assert draft_workflow.updated_by == account.id - sqlite_session.expire_all() - stored = sqlite_session.get(Workflow, draft_workflow.id) - assert stored is not None - assert stored.graph_dict == source_graph - retire_unowned.assert_called_once_with( - tenant_id=snippet.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) - - -def test_restore_published_snippet_workflow_to_draft_raises_when_source_missing( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], -) -> None: - snippet = _snippet() - account = _account("account-2") - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=None)) - - with pytest.raises(WorkflowNotFoundError): - service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id="missing-workflow", - account=account, - ) - - -def test_restore_published_snippet_workflow_to_draft_adds_new_draft( - monkeypatch: pytest.MonkeyPatch, - sqlite_session_factory: sessionmaker[Session], - sqlite_session: Session, -) -> None: - snippet = _snippet() - account = _account("account-2") - source_workflow = _create_workflow( - workflow_id="published-workflow", - version="2026-04-28 00:00:00", - graph={"nodes": [{"id": "llm-1", "data": {"type": "llm"}}], "edges": []}, - features={}, - ) - new_draft_workflow = _create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - graph={"nodes": [], "edges": []}, - features={}, - ) - service = SnippetService(session_maker=sqlite_session_factory) - - monkeypatch.setattr(service, "get_published_workflow_by_id", Mock(return_value=source_workflow)) - monkeypatch.setattr(service, "get_draft_workflow", Mock(return_value=None)) - monkeypatch.setattr( - "services.snippet_service.apply_published_workflow_snapshot_to_draft", - Mock(return_value=(new_draft_workflow, True)), - ) - - result = service.restore_published_workflow_to_draft( - snippet=snippet, - workflow_id=source_workflow.id, - account=account, - ) - - assert result is new_draft_workflow - sqlite_session.expire_all() - assert sqlite_session.get(Workflow, new_draft_workflow.id) is not None - - def test_get_published_workflow_returns_none_without_workflow_id() -> None: service = SnippetService.__new__(SnippetService) @@ -645,7 +402,7 @@ def test_get_published_workflow_by_id_raises_for_draft( ) sqlite_session.add(draft_workflow) sqlite_session.commit() - service = SnippetService(session_maker=sqlite_session_factory) + service = build_snippet_service(session_maker=sqlite_session_factory) with pytest.raises(IsDraftWorkflowError): service.get_published_workflow_by_id( @@ -654,12 +411,11 @@ def test_get_published_workflow_by_id_raises_for_draft( ) -def test_publish_workflow_raises_when_draft_missing(sqlite_session: Session) -> None: - service = SnippetService.__new__(SnippetService) +def test_publish_workflow_raises_when_draft_missing(sqlite_session_factory: sessionmaker[Session]) -> None: + service = build_snippet_service(session_maker=sqlite_session_factory) with pytest.raises(ValueError, match="No valid workflow found"): service.publish_workflow( - session=sqlite_session, snippet=_snippet(), account=_account(), ) @@ -668,7 +424,7 @@ def test_publish_workflow_raises_when_draft_missing(sqlite_session: Session) -> def test_publish_workflow_creates_snapshot_and_updates_snippet( monkeypatch: pytest.MonkeyPatch, sqlite_session: Session ) -> None: - service = SnippetService.__new__(SnippetService) + service = build_snippet_service(session=sqlite_session) draft_workflow = _create_workflow( workflow_id="draft-workflow", version=Workflow.VERSION_DRAFT, @@ -689,25 +445,25 @@ def test_publish_workflow_creates_snapshot_and_updates_snippet( ) snippet = _snippet() sqlite_session.add_all([draft_workflow, snippet]) - sqlite_session.flush() + sqlite_session.commit() monkeypatch.setattr( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", + "services.agent.workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", Mock(return_value=set()), ) result = service.publish_workflow( - session=sqlite_session, snippet=snippet, account=_account(), ) + sqlite_session.refresh(snippet) assert result.kind == WorkflowKind.SNIPPET assert snippet.version == 2 assert snippet.is_published is True assert snippet.workflow_id == result.id assert snippet.updated_by == "account-1" sqlite_session.flush() - assert sqlite_session.get(Workflow, result.id) is result + assert sqlite_session.get(Workflow, result.id) is not None assert sqlite_session.get(CustomizedSnippet, snippet.id).workflow_id == result.id @@ -828,11 +584,11 @@ def test_delete_snippet_releases_last_owner_and_retries_archived_agent_cleanup_a cleanup_delay = Mock(side_effect=lambda **_kwargs: events.append("cleanup-hidden-app")) enqueue_collection = Mock(side_effect=lambda **_kwargs: events.append("enqueue-agent-purge")) monkeypatch.setattr( - "services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", + "services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_delay, ) monkeypatch.setattr( - "services.agent.legacy_retirement_service.enqueue_agent_resource_collection", + "services.agent.retirement_service.enqueue_agent_resource_collection", enqueue_collection, ) @@ -926,7 +682,7 @@ def test_delete_snippet_keeps_agent_with_persisted_external_owner( sqlite_session.flush() cleanup_delay = Mock() monkeypatch.setattr( - "services.agent.legacy_retirement_service.remove_app_and_related_data_task.delay", + "services.agent.retirement_service.remove_app_and_related_data_task.delay", cleanup_delay, ) @@ -1010,9 +766,9 @@ def test_delete_archived_workflow_run_files_removes_prefixed_objects(monkeypatch def test_workflow_run_queries_delegate_to_repositories( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, *, tool_providers ) -> None: - service = SnippetService(session=sqlite_session) + service = build_snippet_service(session=sqlite_session) workflow_run_repo = SimpleNamespace( get_paginated_workflow_runs=Mock(return_value=SimpleNamespace(data=[])), get_workflow_run_by_id=Mock(return_value=WorkflowRun(id="run-1")), @@ -1030,7 +786,10 @@ def test_workflow_run_queries_delegate_to_repositories( assert service.get_snippet_workflow_runs(snippet=snippet, args={"limit": "5", "last_id": "run-0"}).data == [] assert service.get_snippet_workflow_run(snippet=snippet, run_id="run-1").id == "run-1" - assert service.get_snippet_workflow_run_node_executions(snippet=snippet, run_id="run-1") == expected_traces + assert ( + service.get_snippet_workflow_run_node_executions(snippet=snippet, run_id="run-1", tool_providers=tool_providers) + == expected_traces + ) assert ( service.get_snippet_node_last_run( snippet=snippet, @@ -1053,7 +812,10 @@ def test_workflow_run_queries_delegate_to_repositories( workflow_run_id="run-1", ) mock_assemble.assert_called_once_with( - node_execution_repo.get_executions_by_workflow_run.return_value, node_execution_repo, session=sqlite_session + node_execution_repo.get_executions_by_workflow_run.return_value, + node_execution_repo, + session=ANY, + tool_providers=tool_providers, ) node_execution_repo.get_node_last_execution.assert_called_once_with( tenant_id="tenant-1", @@ -1063,14 +825,13 @@ def test_workflow_run_queries_delegate_to_repositories( ) -def test_workflow_run_node_executions_returns_empty_when_run_missing() -> None: +def test_workflow_run_node_executions_returns_empty_when_run_missing(*, tool_providers) -> None: service = SnippetService.__new__(SnippetService) service._node_execution_service_repo = SimpleNamespace(get_executions_by_workflow_run=Mock()) service.get_snippet_workflow_run = Mock(return_value=None) result = service.get_snippet_workflow_run_node_executions( - snippet=_snippet(), - run_id="missing-run", + snippet=_snippet(), run_id="missing-run", tool_providers=tool_providers ) assert result == [] @@ -1088,3 +849,174 @@ def test_increment_use_count_adds_updated_snippet(sqlite_session: Session) -> No assert snippet.use_count == 3 sqlite_session.flush() assert sqlite_session.get(CustomizedSnippet, snippet.id).use_count == 3 + + +@pytest.mark.parametrize("concurrent_edit", [None, "graph", "binding", "agent"]) +def test_snippet_skill_validation_releases_connections_and_rechecks_state( + sqlite_session_factory: sessionmaker[Session], monkeypatch: pytest.MonkeyPatch, concurrent_edit: str | None +) -> None: + import io + import zipfile + + from sqlalchemy import Select, func + from sqlalchemy.dialects import postgresql + from sqlalchemy.engine import Engine + from sqlalchemy.orm import ORMExecuteState + from sqlalchemy.pool import QueuePool + + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource + from models.agent_config_entities import AgentSoulConfig, AgentSoulModelConfig, AgentSoulPromptConfig + from models.skill import AgentSkillBindingSnapshot, Skill, SkillVersion, SkillVersionManifest + from models.tools import ToolFile + + soul = AgentSoulConfig( + model=AgentSoulModelConfig(plugin_id="langgenius/openai", model_provider="openai", model="test"), + prompt=AgentSoulPromptConfig(system_prompt="Use [§skill:archived-skill§]"), + ) + with sqlite_session_factory.begin() as session: + draft = _create_workflow(workflow_id="draft", version="draft", graph={}, features={}) + session.add_all([draft, _snippet()]) + draft.graph = json.dumps( + { + "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}], + "edges": [], + } + ) + archive = ToolFile( + tenant_id="tenant-1", + user_id="account-1", + conversation_id=None, + file_key="skills/published.zip", + mimetype="application/zip", + ) + session.add_all( + [ + archive, + Agent( + id="agent", + tenant_id="tenant-1", + name="Agent", + scope=AgentScope.ROSTER, + source=AgentSource.ROSTER, + active_config_snapshot_id="soul", + ), + AgentConfigSnapshot(id="soul", tenant_id="tenant-1", agent_id="agent", version=1, config_snapshot=soul), + AgentConfigSnapshot( + id="new-soul", tenant_id="tenant-1", agent_id="agent", version=2, config_snapshot=soul + ), + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="snippet-1", + workflow_id="draft", + workflow_version="draft", + node_id="agent", + binding_type=WorkflowAgentBindingType.ROSTER_AGENT, + agent_id="agent", + current_snapshot_id="soul", + node_job_config={}, + created_by="account-1", + ), + Skill( + id="skill", + tenant_id="tenant-1", + name="draft-name", + display_name="Skill", + latest_published_version_id="version", + ), + SkillVersion( + id="version", + skill_id="skill", + version_number=1, + manifest=SkillVersionManifest(files=[]), + archive_tool_file_id=archive.id, + hash_code="hash", + archive_size=123, + ), + AgentSkillBindingSnapshot( + tenant_id="tenant-1", agent_id="agent", config_snapshot_id="soul", skill_id="skill", priority=0 + ), + ] + ) + archive_bytes = io.BytesIO() + with zipfile.ZipFile(archive_bytes, "w") as zip_file: + zip_file.writestr("SKILL.md", "---\nname: archived-skill\ndescription: Published skill\n---\nInstructions") + sessions: list[Session] = [] + statements: list[str] = [] + downloads: list[str] = [] + + def opened(session: Session, _transaction: object, _connection: object) -> None: + sessions.append(session) + + def query(execution: ORMExecuteState) -> None: + if isinstance(execution.statement, Select): + statements.append(str(execution.statement.compile(dialect=postgresql.dialect()))) + + def load(key: str) -> bytes: + assert all(not session.in_transaction() for session in sessions) + engine = sqlite_session_factory.kw["bind"] + assert isinstance(engine, Engine) + assert isinstance(engine.pool, QueuePool) + assert engine.pool.checkedout() == 0 + assert not any("FOR UPDATE" in statement for statement in statements) + downloads.append(key) + if concurrent_edit: + with sqlite_session_factory.begin() as session: + if concurrent_edit == "graph": + draft = session.get(Workflow, "draft") + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + elif concurrent_edit == "binding": + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + else: + agent = session.get(Agent, "agent") + assert agent is not None + agent.active_config_snapshot_id = "new-soul" + return archive_bytes.getvalue() + + monkeypatch.setattr("services.skill_management_service.storage.load_once", load) + service = build_snippet_service(session_maker=sqlite_session_factory) + event.listen(sqlite_session_factory, "after_begin", opened) + event.listen(sqlite_session_factory, "do_orm_execute", query) + try: + if concurrent_edit: + with pytest.raises(WorkflowHashNotEqualError): + service.publish_workflow(snippet=_snippet(), account=_account()) + else: + service.publish_workflow(snippet=_snippet(), account=_account()) + finally: + event.remove(sqlite_session_factory, "after_begin", opened) + event.remove(sqlite_session_factory, "do_orm_execute", query) + assert downloads == ["skills/published.zip"] + with sqlite_session_factory() as session: + snippet = session.get(CustomizedSnippet, "snippet-1") + assert snippet is not None + assert snippet.is_published is (concurrent_edit is None) + versions = session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) + assert versions == (0 if concurrent_edit else 1) + + +def test_snippet_publication_rolls_back_version_and_pointer_on_binding_write_failure( + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from sqlalchemy import func + + with sqlite_session_factory.begin() as session: + session.add_all([_snippet(), _create_workflow(workflow_id="draft", version="draft", graph={}, features={})]) + monkeypatch.setattr( + "services.agent.workflow_publish_service.WorkflowAgentPublishService.copy_agent_node_bindings_to_published", + Mock(side_effect=ValueError("binding write failed")), + ) + service = build_snippet_service(session_maker=sqlite_session_factory) + with pytest.raises(ValueError, match="binding write failed"): + service.publish_workflow(snippet=_snippet(), account=_account()) + with sqlite_session_factory() as session: + snippet = session.get(CustomizedSnippet, "snippet-1") + assert snippet is not None + assert snippet.workflow_id is None + assert snippet.is_published is False + assert snippet.version == 1 + assert session.scalar(select(func.count()).select_from(Workflow).where(Workflow.version != "draft")) == 0 diff --git a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py index 5e5ce5b4ea646e..529deca6e8296a 100644 --- a/api/tests/unit_tests/services/test_trial_app_generation_adapters.py +++ b/api/tests/unit_tests/services/test_trial_app_generation_adapters.py @@ -8,13 +8,16 @@ from sqlalchemy.orm import Session, sessionmaker from sqlalchemy.pool import QueuePool -from core.app.apps.chat.app_generator import ChatAppGenerator from core.app.entities.app_invoke_entities import InvokeFrom +from extensions.application_services.workflow import WorkflowExecutionDependencies from models import Account, App, AppMode, Conversation from models.enums import ConversationFromSource from models.model import AccountTrialAppRecord from repositories.trial_app_repository import TrialAppRepository from services.account_errors import AccountNotFoundError +from services.app.generation.adapters.chat import ChatAppGenerator +from services.app.generation.response import convert_to_event_stream +from services.app.generation.runtime import AppGenerationRuntime from services.app_definition_query_service import AppDefinitionUnavailableError from services.app_generate_service import AppGenerateService from services.errors.app_model_config import AppModelConfigBrokenError @@ -22,6 +25,7 @@ from services.trial_app_access_service import TrialAppRef from services.trial_app_generation_adapters import AppGenerateServiceRuntime from services.trial_app_generation_service import GenerationResponse, TrialAppGenerationService +from services.workflow.variable_contracts import WorkflowExecutionVariables _ARGS: dict[str, object] = {"inputs": {"count": 0}, "query": "hello", "auto_generate_name": False} @@ -36,7 +40,13 @@ class _Harness: @pytest.fixture -def harness(sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session]) -> _Harness: +def harness( + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, + workflow_runtime: WorkflowExecutionDependencies, +) -> _Harness: with sqlite_session_factory.begin() as session: app = App(tenant_id=str(uuid4()), name="Trial app", mode=AppMode.CHAT, enable_site=True, enable_api=False) account = Account(name="Viewer", email="viewer@example.com") @@ -64,7 +74,9 @@ def close(self) -> None: factory = sessionmaker(bind=sqlite_engine, class_=TrackedSession, expire_on_commit=True) return _Harness( - runtime=AppGenerateServiceRuntime(session_factory=cast(sessionmaker[Session], factory)), + runtime=AppGenerateServiceRuntime( + session_factory=cast(sessionmaker[Session], factory), variables=workflow_variables, runtime=workflow_runtime + ), app=TrialAppRef(app_id=app.id, tenant_id=app.tenant_id, app_mode=app.mode), account_id=account.id, closed_sessions=closed_sessions, @@ -103,8 +115,10 @@ def _patch_generation( expected_streaming = streaming expected_args = args - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, @@ -112,6 +126,8 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert len(harness.closed_sessions) == 1 read_session = harness.closed_sessions[0] assert not read_session.in_transaction() @@ -131,7 +147,7 @@ def legacy_generate( assert args == expected_args return generate(session) - monkeypatch.setattr(AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(AppGenerateService, "generate", boundary_generate) def test_mapping_result_is_unchanged_and_generation_writes_are_committed( @@ -351,8 +367,10 @@ def _add_conversation(harness: _Harness, session_factory: sessionmaker[Session]) def _patch_chat_generation(monkeypatch: pytest.MonkeyPatch, harness: _Harness) -> None: - def legacy_generate( + def boundary_generate( *, + variables: WorkflowExecutionVariables, + runtime: AppGenerationRuntime, session: Session, app_model: App, user: Account, @@ -360,16 +378,22 @@ def legacy_generate( invoke_from: InvokeFrom, streaming: bool, ) -> GenerationResponse: + assert runtime is harness.runtime._runtime + assert variables is harness.runtime._variables assert len(harness.closed_sessions) == 1 assert not harness.closed_sessions[0].in_transaction() assert session is not harness.closed_sessions[0] assert not session.in_transaction() - response = ChatAppGenerator().generate( + response = ChatAppGenerator( + retrieval=runtime.retrieval, + annotations=runtime.annotation_replies, + records=runtime.chat_records, + ).generate( app_model=app_model, user=user, args=args, invoke_from=invoke_from, streaming=streaming, session=session ) - return cast(GenerationResponse, ChatAppGenerator.convert_to_event_stream(response)) + return cast(GenerationResponse, convert_to_event_stream(response)) - monkeypatch.setattr(AppGenerateService, "generate", legacy_generate) + monkeypatch.setattr(AppGenerateService, "generate", boundary_generate) @pytest.mark.parametrize("mismatch", ["missing", "app", "account", "source", "end-user", "deleted"]) diff --git a/api/tests/unit_tests/services/test_workflow_app_log_query_service.py b/api/tests/unit_tests/services/test_workflow_app_log_query_service.py index f5a3c61492e3d1..085180def073d8 100644 --- a/api/tests/unit_tests/services/test_workflow_app_log_query_service.py +++ b/api/tests/unit_tests/services/test_workflow_app_log_query_service.py @@ -7,8 +7,8 @@ import pytest from models.enums import AppTriggerType -from repositories.workflow_app_log_query_repository import WorkflowAppLogQueryRepository -from services.workflow_app_log_query_service import ( +from repositories.workflow.app_log_repository import WorkflowAppLogRepository +from services.workflow.app_log_query_service import ( WorkflowAppLogItem, WorkflowAppLogPage, WorkflowAppLogQueryService, @@ -60,7 +60,7 @@ def test_list_logs_enriches_plugin_icons() -> None: service = _service_with_metadata(json.dumps(metadata)) with patch( - "services.workflow_app_log_query_service.PluginService.get_plugin_icon_url", + "services.workflow.app_log_query_service.PluginService.get_plugin_icon_url", side_effect=["https://cdn/light.png", "https://cdn/dark.png"], ) as get_icon_url: result = service.list_logs(tenant_id="tenant-1", app_id="app-1", detail=True) @@ -76,7 +76,7 @@ def test_list_logs_enriches_plugin_icons() -> None: def test_list_logs_does_not_fetch_icons_for_non_plugin_metadata() -> None: service = _service_with_metadata(json.dumps({"type": AppTriggerType.TRIGGER_WEBHOOK.value})) - with patch("services.workflow_app_log_query_service.PluginService.get_plugin_icon_url") as get_icon_url: + with patch("services.workflow.app_log_query_service.PluginService.get_plugin_icon_url") as get_icon_url: result = service.list_logs(tenant_id="tenant-1", app_id="app-1", detail=True) assert result.data[0].details == {"trigger_metadata": {"type": AppTriggerType.TRIGGER_WEBHOOK.value}} @@ -106,14 +106,14 @@ def test_safe_json_loads(value: object, expected: object) -> None: def test_safe_parse_uuid_rejects_short_and_invalid_values() -> None: - assert WorkflowAppLogQueryRepository._safe_parse_uuid("short") is None - assert WorkflowAppLogQueryRepository._safe_parse_uuid("x" * 40) is None + assert WorkflowAppLogRepository._safe_parse_uuid("short") is None + assert WorkflowAppLogRepository._safe_parse_uuid("x" * 40) is None def test_safe_parse_uuid_accepts_uuid() -> None: raw = str(uuid.uuid4()) - result = WorkflowAppLogQueryRepository._safe_parse_uuid(raw) + result = WorkflowAppLogRepository._safe_parse_uuid(raw) assert result is not None assert str(result) == raw diff --git a/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py b/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py index 1324c13c376323..03b4c528b16aed 100644 --- a/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py +++ b/api/tests/unit_tests/services/test_workflow_node_execution_trace_service.py @@ -78,7 +78,7 @@ def _repository(full_process_data: dict[str, object] | None) -> DifyAPIWorkflowN return cast(DifyAPIWorkflowNodeExecutionRepository, repository) -def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Session) -> None: +def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Session, *, tool_providers) -> None: process_data = { "request": "successful-attempt", RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(2), _retry_attempt(1)], @@ -86,7 +86,9 @@ def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Se execution = _execution(process_data) repository = _repository(process_data) - traces = assemble_workflow_node_execution_traces([execution], repository, session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], repository, session=sqlite_session, tool_providers=tool_providers + ) assert [trace.id for trace in traces] == ["exec-1:retry:1", "exec-1:retry:2", "exec-1"] assert [trace.status for trace in traces] == ["retry", "retry", "succeeded"] @@ -105,18 +107,20 @@ def test_assemble_expands_retry_history_before_terminal_trace(sqlite_session: Se repository.load_full_process_data.assert_called_once_with(execution) -def test_assemble_keeps_old_execution_without_retry_history(sqlite_session: Session) -> None: +def test_assemble_keeps_old_execution_without_retry_history(sqlite_session: Session, *, tool_providers) -> None: process_data = {"request": "terminal"} execution = _execution(process_data) - traces = assemble_workflow_node_execution_traces([execution], _repository(process_data), session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], _repository(process_data), session=sqlite_session, tool_providers=tool_providers + ) assert len(traces) == 1 assert traces[0].id == "exec-1" assert traces[0].process_data == process_data -def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: Session) -> None: +def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: Session, *, tool_providers) -> None: process_data = { RETRY_HISTORY_PROCESS_DATA_KEY: [ _retry_attempt(2), @@ -127,39 +131,43 @@ def test_assemble_skips_malformed_and_duplicate_retry_attempts(sqlite_session: S } traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) assert [trace.retry_index for trace in traces[:-1]] == [1, 2] assert traces[0].error == "attempt 1 failed" -def test_assemble_truncates_retry_attempt_fields(sqlite_session: Session) -> None: +def test_assemble_truncates_retry_attempt_fields(sqlite_session: Session, *, tool_providers) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1, outputs={"body": "x" * 2_000_000})]} traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) assert traces[0].outputs_truncated is True assert traces[-1].id == "exec-1" -def test_assemble_falls_back_to_inline_process_data_when_loader_fails(sqlite_session: Session) -> None: +def test_assemble_falls_back_to_inline_process_data_when_loader_fails( + sqlite_session: Session, *, tool_providers +) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1)]} execution = _execution(process_data) repository = _repository(None) repository.load_full_process_data.side_effect = OSError("storage unavailable") - traces = assemble_workflow_node_execution_traces([execution], repository, session=sqlite_session) + traces = assemble_workflow_node_execution_traces( + [execution], repository, session=sqlite_session, tool_providers=tool_providers + ) assert [trace.id for trace in traces] == ["exec-1:retry:1", "exec-1"] -def test_virtual_trace_validates_through_node_execution_response(sqlite_session: Session) -> None: +def test_virtual_trace_validates_through_node_execution_response(sqlite_session: Session, *, tool_providers) -> None: process_data = {RETRY_HISTORY_PROCESS_DATA_KEY: [_retry_attempt(1)]} traces = assemble_workflow_node_execution_traces( - [_execution(process_data)], _repository(process_data), session=sqlite_session + [_execution(process_data)], _repository(process_data), session=sqlite_session, tool_providers=tool_providers ) response = WorkflowRunNodeExecutionListResponse.model_validate({"data": traces}, from_attributes=True) diff --git a/api/tests/unit_tests/services/test_workflow_run_agg.py b/api/tests/unit_tests/services/test_workflow_run_agg.py index 68b3cb4b2f912b..463c90da068312 100644 --- a/api/tests/unit_tests/services/test_workflow_run_agg.py +++ b/api/tests/unit_tests/services/test_workflow_run_agg.py @@ -9,8 +9,6 @@ from sqlalchemy.orm import Session, sessionmaker from clients.agent_backend import FakeAgentBackendRunClient, FakeAgentBackendScenario -from core.app.apps.pipeline.pipeline_queue_manager import PipelineQueueManager -from core.app.apps.pipeline.pipeline_runner import PipelineRunner from core.app.apps.workflow.app_config_manager import WorkflowAppConfig as CoreWorkflowAppConfig from core.app.apps.workflow.app_queue_manager import WorkflowAppQueueManager as CoreWorkflowAppQueueManager from core.app.apps.workflow.app_runner import WorkflowAppRunner as CoreWorkflowAppRunner @@ -64,6 +62,8 @@ from repositories.workflow.definition_repository import workflow_snapshot from services.agent.workspace_service import AgentWorkspaceService from services.workflow.entities import WorkflowScheduleCFSPlanEntity +from services.workflow.execution.adapters.pipeline.pipeline_queue_manager import PipelineQueueManager +from services.workflow.execution.adapters.pipeline.pipeline_runner import PipelineRunner from services.workflow.execution.adapters.workflow.app_config_manager import WorkflowAppConfig from services.workflow.execution.adapters.workflow.app_queue_manager import WorkflowAppQueueManager from services.workflow.execution.adapters.workflow.app_runner import WorkflowAppRunner @@ -85,6 +85,7 @@ def _pipeline_tool_runner( sqlite_engine: Engine, + workflow_runtime: WorkflowExecutionDependencies, invoke_from: InvokeFrom, source_node_data: dict[str, object], ) -> tuple[PipelineRunner, SQLAlchemyWorkflowExecutionRepository, SQLAlchemyWorkflowNodeExecutionRepository]: @@ -167,9 +168,10 @@ def _pipeline_tool_runner( app_mode=AppMode.RAG_PIPELINE, ), variable_loader=DUMMY_VARIABLE_LOADER, - workflow=workflow, + workflow=workflow_snapshot(workflow), system_user_id="sys", documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)), + runtime=workflow_runtime, ) user = Account(name="User", email="pipeline@example.com") user.id = entity.user_id @@ -192,12 +194,19 @@ class TestPipelineWorkflowToolsCases: @staticmethod @pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.PUBLISHED_PIPELINE]) def test_pipeline_rejects_human_input_tool_before_creating_run_or_form( - sqlite_engine: Engine, invoke_from: InvokeFrom + sqlite_engine: Engine, + workflow_runtime: WorkflowExecutionDependencies, + invoke_from: InvokeFrom, ) -> None: human_input = HumanInputNodeData( title="Approval", user_actions=[UserActionConfig(id="approve", title="Approve")] ) - runner, runs, nodes = _pipeline_tool_runner(sqlite_engine, invoke_from, human_input.model_dump(mode="json")) + runner, runs, nodes = _pipeline_tool_runner( + sqlite_engine, + workflow_runtime, + invoke_from, + human_input.model_dump(mode="json"), + ) with pytest.raises(ValueError, match="Human Input nodes are not supported"): WorkflowRunAgg.run(runner, None, runs, nodes) @@ -223,6 +232,7 @@ def test_pipeline_rejects_human_input_tool_before_creating_run_or_form( ) def test_pipeline_retires_nested_agent_workspace_when_run_ends( sqlite_engine: Engine, + workflow_runtime: WorkflowExecutionDependencies, monkeypatch: pytest.MonkeyPatch, invoke_from: InvokeFrom, scenario: FakeAgentBackendScenario, @@ -230,6 +240,7 @@ def test_pipeline_retires_nested_agent_workspace_when_run_ends( ) -> None: runner, runs, nodes = _pipeline_tool_runner( sqlite_engine, + workflow_runtime, invoke_from, {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"}, ) @@ -260,7 +271,8 @@ def test_pipeline_retires_nested_agent_workspace_when_run_ends( lambda **_kwargs: FakeAgentBackendRunClient(scenario=scenario), ) monkeypatch.setattr( - "core.workflow.nodes.agent_v2.runtime_request_builder.resolve_model_context_window", lambda **_kwargs: None + "services.workflow.execution.adapters.agent_v2.runtime_request_builder.resolve_model_context_window", + lambda **_kwargs: None, ) collections: list[dict[str, object]] = [] @@ -746,15 +758,23 @@ def test_durable_pause_resumes_with_one_form_completion_and_continuing_indexes( @staticmethod @pytest.mark.usefixtures("_local_storage") - def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_engine: Engine) -> None: + def test_pause_before_first_tool_call_preserves_its_published_definition( + sqlite_engine: Engine, + sqlite_session_factory: sessionmaker[Session], + workflow_runtime: WorkflowExecutionDependencies, + ) -> None: from core.app.entities.queue_entities import QueueHumanInputFormFilledEvent - from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from graphon.nodes.tool.entities import ToolNodeData from models.human_input import HumanInputForm from models.model import App from models.tools import WorkflowToolProvider + from repositories.human_input.form_repository import HumanInputFormSubmissionRepository - original, workflow_repository, node_repository = _make_legacy_workflow_runner(sqlite_engine, human_input=True) + original, workflow_repository, node_repository = make_workflow_runner( + sqlite_engine, + workflow_runtime, + human_input=True, + ) original_entity = original.application_generate_entity tenant_id = original_entity.app_config.tenant_id source_app_id = str(uuid4()) @@ -853,7 +873,7 @@ def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_ graph["nodes"][2]["data"]["outputs"] = [{"variable": "result", "value_selector": ["tool", "json"]}] graph["edges"][1]["target"] = "tool" graph["edges"].append({"source": "tool", "target": "end"}) - original._workflow.graph = json.dumps(graph) + original._workflow = replace(original._workflow, graph=json.dumps(graph)) config = PauseStateLayerConfig(sqlite_engine, original_entity.user_id) WorkflowRunAgg.run(original, config, workflow_repository, node_repository) before = [message.event for message in original._queue_manager.listen()] @@ -870,7 +890,7 @@ def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_ snapshot = WorkflowResumptionContext.loads(storage.load(pause.state_object_key).decode()) provider.version = "new" provider.label = "Republished source" - HumanInputFormSubmissionRepository().mark_submitted( + HumanInputFormSubmissionRepository(sessions=sqlite_session_factory).mark_submitted( form_id=form_id, recipient_id=None, selected_action_id="approve", @@ -880,8 +900,9 @@ def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_ ) entity = snapshot.get_generate_entity().model_copy(update={"task_id": str(uuid4())}) assert isinstance(entity, WorkflowAppGenerateEntity) - queue = CoreWorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW) - resumed = CoreWorkflowAppRunner( + queue = WorkflowAppQueueManager(entity.task_id, entity.user_id, entity.invoke_from, AppMode.WORKFLOW) + resumed = WorkflowAppRunner( + runtime=workflow_runtime, application_generate_entity=entity, queue_manager=queue, variable_loader=DUMMY_VARIABLE_LOADER, @@ -889,7 +910,6 @@ def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_ system_user_id=entity.user_id, graph_runtime_state=RuntimeState.from_snapshot(snapshot.serialized_graph_runtime_state), response_stream_filter=snapshot.get_response_stream_filter(), - execution_driver=WorkflowRunAgg.run, ) WorkflowRunAgg.run(resumed, config, workflow_repository, node_repository) after = [message.event for message in queue.listen()] @@ -907,20 +927,20 @@ def test_pause_before_first_tool_call_preserves_its_published_definition(sqlite_ # A new execution resolves the newly published graph and parameter defaults. graph["nodes"] = [node for node in graph["nodes"] if node["id"] != "approval"] graph["edges"] = [{"source": "start", "target": "tool"}, {"source": "tool", "target": "end"}] - original._workflow.graph = json.dumps(graph) + original._workflow = replace(original._workflow, graph=json.dumps(graph)) fresh_entity = original_entity.model_copy( update={"task_id": str(uuid4()), "workflow_execution_id": str(uuid4()), "extras": {}} ) - fresh_queue = CoreWorkflowAppQueueManager( + fresh_queue = WorkflowAppQueueManager( fresh_entity.task_id, fresh_entity.user_id, fresh_entity.invoke_from, AppMode.WORKFLOW ) - fresh = CoreWorkflowAppRunner( + fresh = WorkflowAppRunner( + runtime=workflow_runtime, application_generate_entity=fresh_entity, queue_manager=fresh_queue, variable_loader=DUMMY_VARIABLE_LOADER, workflow=original._workflow, system_user_id=fresh_entity.user_id, - execution_driver=WorkflowRunAgg.run, ) WorkflowRunAgg.run(fresh, None, workflow_repository, node_repository) fresh_events = [message.event for message in fresh_queue.listen()] diff --git a/api/tests/unit_tests/services/test_workflow_run_service.py b/api/tests/unit_tests/services/test_workflow_run_service.py index 011a1a743f8b37..092ee0aa7e1472 100644 --- a/api/tests/unit_tests/services/test_workflow_run_service.py +++ b/api/tests/unit_tests/services/test_workflow_run_service.py @@ -3,7 +3,7 @@ from unittest.mock import ANY, MagicMock, create_autospec import pytest -from pydantic import TypeAdapter +from pydantic import SecretStr, TypeAdapter from sqlalchemy.orm import Session, sessionmaker from core.workflow.node_execution_process_data import ( @@ -41,12 +41,13 @@ """Unit tests for the Console workflow-run application service.""" -def _workflow_run_service_service(dependencies: tuple[MagicMock, MagicMock]) -> WorkflowRunService: +def _workflow_run_service_service(dependencies: tuple[MagicMock, MagicMock], *, tool_providers) -> WorkflowRunService: node_executions, workflow_runs = dependencies return WorkflowRunService( workflow_runs=workflow_runs, session_factory=sessionmaker(), node_executions=node_executions, + tool_providers=tool_providers, ) @@ -86,10 +87,14 @@ def service_dependencies() -> tuple[MagicMock, MagicMock]: class TestWorkflowRunServiceQueries: def test_workflow_tool_details_require_the_root_app_run_and_hide_internal_correlation( - self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch + self, + service_dependencies: tuple[MagicMock, MagicMock], + monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: node_executions, workflow_runs = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) workflow_runs.get_workflow_run_by_id.return_value = None context = _workflow_run_service_request_context() assert ( @@ -126,7 +131,10 @@ def test_workflow_tool_details_require_the_root_app_run_and_hide_internal_correl tenant_id="tenant-1", workflow_run_id="run-1", parent_node_execution_id="parent-1" ) assemble.assert_called_once_with( - node_executions.get_workflow_tool_executions.return_value, node_executions, session=ANY + node_executions.get_workflow_tool_executions.return_value, + node_executions, + session=ANY, + tool_providers=tool_providers, ) assert result == [trace] assert trace.process_data == {"answer": "approved"} @@ -134,9 +142,11 @@ def test_workflow_tool_details_require_the_root_app_run_and_hide_internal_correl def test_get_paginate_workflow_runs_should_forward_filters_and_parse_limit( self, service_dependencies: tuple[MagicMock, MagicMock], + *, + tool_providers, ) -> None: _, workflow_runs = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) expected = MagicMock(name="pagination") workflow_runs.get_paginated_workflow_runs.return_value = expected args = TypeAdapter(WorkflowRunListArgs).validate_python( @@ -164,9 +174,11 @@ def test_get_paginate_advanced_chat_workflow_runs_should_attach_message_fields_w self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: _, workflow_runs = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) run_with_message = _workflow_run(status=WorkflowExecutionStatus.RUNNING) run_without_message = _workflow_run(run_id="run-2") pagination = SimpleNamespace(data=[run_with_message, run_without_message]) @@ -196,9 +208,11 @@ def test_get_paginate_advanced_chat_workflow_runs_should_attach_message_fields_w def test_get_workflow_run_should_delegate_to_repository_by_tenant_and_app( self, service_dependencies: tuple[MagicMock, MagicMock], + *, + tool_providers, ) -> None: _, workflow_runs = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) expected = _workflow_run() workflow_runs.get_workflow_run_by_id.return_value = expected @@ -214,9 +228,11 @@ def test_get_workflow_run_should_delegate_to_repository_by_tenant_and_app( def test_get_workflow_runs_count_should_forward_optional_filters( self, service_dependencies: tuple[MagicMock, MagicMock], + *, + tool_providers, ) -> None: _, workflow_runs = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) expected = {"total": 3, "succeeded": 2} workflow_runs.get_workflow_runs_count.return_value = expected @@ -241,8 +257,10 @@ def test_get_workflow_run_node_executions_should_return_empty_list_when_run_not_ self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) monkeypatch.setattr(service, "get_workflow_run", MagicMock(return_value=None)) result = service.get_workflow_run_node_executions( @@ -257,9 +275,11 @@ def test_get_workflow_run_node_executions_should_use_request_workspace( self, service_dependencies: tuple[MagicMock, MagicMock], monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: node_executions, _ = service_dependencies - service = _workflow_run_service_service(service_dependencies) + service = _workflow_run_service_service(service_dependencies, tool_providers=tool_providers) monkeypatch.setattr(service, "get_workflow_run", MagicMock(return_value=_workflow_run())) expected_executions = [SimpleNamespace(id="exec-1"), SimpleNamespace(id="exec-2")] expected_traces = [SimpleNamespace(id="exec-1:retry:1"), SimpleNamespace(id="exec-1")] @@ -279,27 +299,9 @@ def test_get_workflow_run_node_executions_should_use_request_workspace( app_id="app-1", workflow_run_id="run-1", ) - mock_assemble.assert_called_once_with(expected_executions, node_executions, session=ANY) - - -"""Tests for Console workflow pause details.""" - - -def _workflow_run_service_pause_service(workflow_runs: MagicMock) -> WorkflowRunService: - return WorkflowRunService( - workflow_runs=workflow_runs, - node_executions=MagicMock(), - session_factory=sessionmaker(), - ) - - -def _workflow_run_service_pause_request_context(*, workspace_id: str = "tenant-1") -> RequestContext: - return RequestContext( - request_id="request-1", - trace_id="trace-1", - account_id="account-1", - active_workspace_id=workspace_id, - ) + mock_assemble.assert_called_once_with( + expected_executions, node_executions, session=ANY, tool_providers=tool_providers + ) """Tests for Console workflow pause details.""" @@ -310,11 +312,12 @@ def workflow_runs() -> DifyAPISQLAlchemyWorkflowRunRepository: return DifyAPISQLAlchemyWorkflowRunRepository(sessionmaker()) -def _service(workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository) -> WorkflowRunService: +def _service(workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, *, tool_providers) -> WorkflowRunService: return WorkflowRunService( workflow_runs=workflow_runs, node_executions=DifyAPISQLAlchemyWorkflowNodeExecutionRepository(sessionmaker()), session_factory=sessionmaker(), + tool_providers=tool_providers, ) @@ -328,7 +331,10 @@ def _request_context(*, workspace_id: str = "tenant-1") -> RequestContext: def test_get_pause_details_returns_none_when_run_is_not_found( - workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, monkeypatch: pytest.MonkeyPatch + workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, + monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: lookups: list[tuple[str, str]] = [] @@ -337,7 +343,9 @@ def missing(_session: Session, *, workspace_id: str, workflow_run_id: str) -> No monkeypatch.setattr(workflow_runs, "get_pause_record", missing) - result = _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) assert result is None assert lookups == [("tenant-1", "run-1")] @@ -348,6 +356,8 @@ def test_get_pause_details_returns_empty_details_without_an_active_pause( workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, status: WorkflowExecutionStatus, monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: record = WorkflowRunPauseRecord( status=status, @@ -355,13 +365,18 @@ def test_get_pause_details_returns_empty_details_without_an_active_pause( ) monkeypatch.setattr(workflow_runs, "get_pause_record", lambda *_args, **_kwargs: record) - result = _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) assert result == WorkflowRunPauseDetails(paused_at=None, paused_nodes=()) def test_get_pause_details_maps_human_input_and_token( - workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, monkeypatch: pytest.MonkeyPatch + workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, + monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: reason = HumanInputRequired( form_id="form-1", @@ -395,7 +410,7 @@ def pause_record(_session: Session, *, workspace_id: str, workflow_run_id: str) ), ) - result = _service(workflow_runs).get_pause_details( + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( _request_context(workspace_id="tenant-context"), workflow_run_id="run-1", ) @@ -416,8 +431,62 @@ def pause_record(_session: Session, *, workspace_id: str, workflow_run_id: str) assert lookups == [("tenant-context", "run-1")] +def test_get_pause_details_maps_v2_form_version_and_initiator_token( + workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, + monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, +) -> None: + reason = HumanInputRequired( + form_id="form-2", + form_content="Approve?", + node_id="node-2", + node_title="V2 Approval", + form_version="2", + ) + paused_at = datetime(2026, 1, 2, 3, 4, 5) + pause_model = WorkflowPause(workflow_id="workflow-1", workflow_run_id="run-1", state_object_key="state") + pause_model.created_at = paused_at + record = WorkflowRunPauseRecord( + status=WorkflowExecutionStatus.PAUSED, + pause=_PrivateWorkflowPauseEntity(pause_model=pause_model), + ) + monkeypatch.setattr(workflow_runs, "get_pause_record", lambda *_args, **_kwargs: record) + legacy_tokens = MagicMock() + monkeypatch.setattr(workflow_runs, "get_legacy_form_tokens", legacy_tokens) + monkeypatch.setattr( + "services.workflow_run_service.load_workflow_pause_snapshot", + lambda *_args, **_kwargs: WorkflowPauseSnapshot( + runtime_state=RuntimeState(workflow_id="workflow", variable_pool=VariablePool(), start_at=0), + reasons=(reason,), + v2_forms={"form-2": SimpleNamespace(form_token=SecretStr("v2-token"))}, + ), + ) + + result = _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) + + assert result == WorkflowRunPauseDetails( + paused_at=paused_at, + paused_nodes=( + WorkflowRunPausedNode( + node_id="node-2", + node_title="V2 Approval", + form_id="form-2", + form_version="2", + form_token="v2-token", + ), + ), + ) + legacy_tokens.assert_not_called() + + def test_get_pause_details_rejects_unsupported_pause_reason( - workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, monkeypatch: pytest.MonkeyPatch + workflow_runs: DifyAPISQLAlchemyWorkflowRunRepository, + monkeypatch: pytest.MonkeyPatch, + *, + tool_providers, ) -> None: record = WorkflowRunPauseRecord( status=WorkflowExecutionStatus.PAUSED, @@ -436,4 +505,6 @@ def test_get_pause_details_rejects_unsupported_pause_reason( ) with pytest.raises(NotImplementedError, match="Pause details do not support SchedulingPause"): - _service(workflow_runs).get_pause_details(_request_context(), workflow_run_id="run-1") + _service(workflow_runs, tool_providers=tool_providers).get_pause_details( + _request_context(), workflow_run_id="run-1" + ) diff --git a/api/tests/unit_tests/services/test_workflow_service.py b/api/tests/unit_tests/services/test_workflow_service.py index 3e3edc82a95a29..a5f71e62d3adbd 100644 --- a/api/tests/unit_tests/services/test_workflow_service.py +++ b/api/tests/unit_tests/services/test_workflow_service.py @@ -1,33 +1,35 @@ +""" +Unit tests for WorkflowService. + +This test suite covers: +- Workflow creation from template +- Workflow validation (graph and features structure) +- Draft/publish transitions +- Version management +- Execution triggering +""" + import json -import os -import time import uuid -from collections.abc import Callable, Generator, Iterator -from datetime import UTC, datetime, timedelta -from types import SimpleNamespace +from collections.abc import Callable, Generator +from dataclasses import replace from typing import Any, cast -from unittest.mock import ANY, MagicMock, patch, sentinel -from uuid import uuid4 +from unittest.mock import ANY, MagicMock, patch import pytest -from flask import Flask from pytest_mock import MockerFixture -from sqlalchemy import Engine as _workflow_agent_preview_identity_Engine -from sqlalchemy import event, func, select +from sqlalchemy import select from sqlalchemy.dialects import postgresql -from sqlalchemy.engine import Engine as _workflow_service_Engine +from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker from core.repositories.celery_workflow_node_execution_repository import CeleryWorkflowNodeExecutionRepository from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository from core.workflow.llm_environment_variable import LLMEnvironmentVariable -from core.workflow.node_factory import DifyNodeFactory -from core.workflow.nodes.agent_v2 import DifyAgentNode from core.workflow.nodes.agent_v2.session_store import WorkflowAgentWorkspaceStore from core.workflow.system_variables import SystemVariableKey, get_system_text from enums import DeploymentEdition -from enums.human_input import RecipientType -from extensions.ext_database import db +from extensions.application_services.workflow import WorkflowExecutionDependencies from extensions.logstore.repositories.logstore_workflow_node_execution_repository import ( LogstoreWorkflowNodeExecutionRepository, ) @@ -43,63 +45,29 @@ from graphon.node_events import NodeRunResult from graphon.nodes.http_request import HTTP_REQUEST_CONFIG_FILTER_KEY, HttpRequestNode, HttpRequestNodeConfig from graphon.runtime import RuntimeState, VariablePool -from graphon.variables import StringVariable from graphon.variables.input_entities import VariableEntityType from libs.datetime_utils import naive_utc_now -from models import Account as _workflow_human_input_delivery_Account -from models.account import Account as _workflow_service_Account -from models.agent import ( - Agent, - AgentConfigSnapshot, - AgentKind, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) -from models.agent_config_entities import AgentSoulConfig -from models.human_input import HumanInputFormRecipient -from models.human_input_delivery import ( - EmailDeliveryConfig, - EmailDeliveryMethod, - EmailRecipients, - ExternalRecipient, - MemberRecipient, -) -from models.human_input_entities import HumanInputNodeData +from machinery.context import RequestContext +from models.account import Account from models.model import App, AppMode -from models.tools import BuiltinToolProvider, WorkflowToolProvider +from models.tools import BuiltinToolProvider from models.workflow import Workflow, WorkflowNodeExecutionModel, WorkflowNodeExecutionTriggeredFrom, WorkflowType -from repositories.factory import DifyAPIRepositoryFactory -from services import workflow_service as workflow_service_module -from services.agent.legacy_retirement_service import WorkflowAgentRetirementService -from services.errors.app import IsDraftWorkflowError, TriggerNodeLimitExceededError, WorkflowHashNotEqualError -from services.errors.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError -from services.workflow_ref_service import WorkflowRef +from repositories.credentials.query_repository import CredentialQueryRepository +from repositories.workflow.definition_repository import workflow_snapshot +from services.errors.app import IsDraftWorkflowError +from services.human_input.debug_service import HumanInputDebugService +from services.human_input.file_gateway import HumanInputDebugFileGateway +from services.workflow.contracts import WorkflowDebugNode +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.variable_contracts import WorkflowExecutionVariables from services.workflow_service import ( WorkflowService, - _merge_environment_variable_patch, _rebuild_file_for_user_inputs_in_start_node, _rebuild_single_file, _setup_variable_pool, ) -from tests.helpers import billing_builders as builders -from tests.helpers.unit.config_override import apply_config_overrides -from tests.helpers.unit.model_factories import make_account, make_app from tests.workflow_test_utils import build_test_graph_init_params -""" -Unit tests for WorkflowService. - -This test suite covers: -- Workflow creation from template -- Workflow validation (graph and features structure) -- Draft/publish transitions -- Version management -- Execution triggering -""" - class TestWorkflowAssociatedDataFactory: """ @@ -177,9 +145,9 @@ def create_workflow( return workflow @staticmethod - def create_account(account_id: str = "user-123", **kwargs: Any) -> _workflow_service_Account: + def create_account(account_id: str = "user-123", **kwargs: Any) -> Account: """Create an Account model with specified attributes.""" - account = _workflow_service_Account(name="Test User", email=f"{account_id}@example.com") + account = Account(name="Test User", email=f"{account_id}@example.com") account.id = account_id for key, value in kwargs.items(): setattr(account, key, value) @@ -248,3286 +216,1595 @@ def create_valid_workflow_graph(include_start: bool = True, include_trigger: boo return {"nodes": nodes, "edges": edges} -def _env(variable_id: str, name: str) -> StringVariable: - return StringVariable(id=variable_id, name=name, value="v", selector=["env", name]) +@pytest.mark.usefixtures("sqlite_session") +class TestWorkflowService: + @pytest.fixture(autouse=True) + def _community_edition(self, config_overrides: Callable[..., None]) -> None: + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + """ + Comprehensive unit tests for WorkflowService methods. + + This test suite covers: + - Workflow creation from template + - Workflow validation (graph and features) + - Draft/publish transitions + - Version management + - Workflow deletion and error handling + """ -class TestWorkflowServiceCases: - @pytest.mark.usefixtures("sqlite_session") - class TestWorkflowService: - @pytest.fixture(autouse=True) - def _community_edition(self, config_overrides: Callable[..., None]) -> None: - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + @pytest.fixture + def workflow_service(self, sqlite_engine: Engine) -> WorkflowService: + """Create a WorkflowService whose repositories use the test SQLite engine.""" + return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - """ - Comprehensive unit tests for WorkflowService methods. - - This test suite covers: - - Workflow creation from template - - Workflow validation (graph and features) - - Draft/publish transitions - - Version management - - Workflow deletion and error handling - """ + # ==================== Workflow Existence Tests ==================== + # These tests verify the service can check if a draft workflow exists - @pytest.fixture - def workflow_service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - """Create a WorkflowService whose repositories use the test SQLite engine.""" - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - def test_get_tenant_app_maintainers_scopes_requested_apps( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_app( - app_id="app-1", tenant_id="tenant-1", maintainer="owner-1" - ), - TestWorkflowAssociatedDataFactory.create_app(app_id="app-2", tenant_id="tenant-1"), - TestWorkflowAssociatedDataFactory.create_app( - app_id="app-3", tenant_id="tenant-2", maintainer="owner-2" - ), - ] - ) - sqlite_session.commit() + # ==================== Get Draft Workflow Tests ==================== + # These tests verify retrieval of draft workflows (version="draft") - assert workflow_service.get_tenant_app_maintainers( - ["app-1", "app-2", "app-3", "missing"], "tenant-1", session=sqlite_session - ) == {"app-1": "owner-1", "app-2": None} + def test_get_draft_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): + """ + Test get_draft_workflow returns draft workflow successfully. - # ==================== Workflow Existence Tests ==================== - # These tests verify the service can check if a draft workflow exists + Draft workflows are the working copy that users edit before publishing. + Each app can have only one draft workflow at a time. + """ + app = TestWorkflowAssociatedDataFactory.create_app() + workflow = TestWorkflowAssociatedDataFactory.create_workflow() + sqlite_session.add(workflow) + sqlite_session.commit() - def test_is_workflow_exist_returns_true(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test is_workflow_exist returns True when draft workflow exists. + result = workflow_service.get_draft_workflow(app, session=sqlite_session) - Verifies that the service correctly identifies when an app has a draft workflow. - This is used to determine whether to create or update a workflow. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() + assert result is workflow - result = workflow_service.is_workflow_exist(app, session=sqlite_session) + def test_get_draft_workflow_uses_provided_session(self, workflow_service: WorkflowService, sqlite_session: Session): + """Test get_draft_workflow can reuse an injected SQLAlchemy session.""" + app = TestWorkflowAssociatedDataFactory.create_app() + workflow = TestWorkflowAssociatedDataFactory.create_workflow() + sqlite_session.add(workflow) + sqlite_session.commit() - assert result is True + result = workflow_service.get_draft_workflow(app, session=sqlite_session) - def test_is_workflow_exist_returns_false(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test is_workflow_exist returns False when no draft workflow exists.""" - app = TestWorkflowAssociatedDataFactory.create_app() + assert result is workflow - result = workflow_service.is_workflow_exist(app, session=sqlite_session) + def test_get_draft_workflow_returns_none(self, workflow_service: WorkflowService, sqlite_session: Session): + """Test get_draft_workflow returns None when no draft exists.""" + app = TestWorkflowAssociatedDataFactory.create_app() - assert result is False + result = workflow_service.get_draft_workflow(app, session=sqlite_session) - # ==================== Get Draft Workflow Tests ==================== - # These tests verify retrieval of draft workflows (version="draft") + assert result is None - def test_get_draft_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test get_draft_workflow returns draft workflow successfully. + def test_get_draft_workflow_with_workflow_id(self, workflow_service: WorkflowService, sqlite_session: Session): + """Test get_draft_workflow with workflow_id calls get_published_workflow_by_id.""" + app = TestWorkflowAssociatedDataFactory.create_app() + workflow_id = "workflow-123" + workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") + sqlite_session.add(workflow) + sqlite_session.commit() - Draft workflows are the working copy that users edit before publishing. - Each app can have only one draft workflow at a time. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() + result = workflow_service.get_draft_workflow(app, workflow_id=workflow_id, session=sqlite_session) - result = workflow_service.get_draft_workflow(app, session=sqlite_session) + assert result is workflow - assert result is workflow + def test_get_draft_workflow_with_workflow_id_reuses_provided_session( + self, workflow_service: WorkflowService, sqlite_session: Session + ): + """A version lookup sees the caller's uncommitted rows through the shared repository.""" + app = TestWorkflowAssociatedDataFactory.create_app() + workflow_id = "workflow-123" + workflow = TestWorkflowAssociatedDataFactory.create_workflow( + workflow_id=workflow_id, app_id=app.id, tenant_id=app.tenant_id, version="v1" + ) + sqlite_session.add(workflow) + assert workflow_service.get_draft_workflow(app, workflow_id=workflow_id, session=sqlite_session) is workflow - def test_get_draft_workflow_uses_provided_session( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test get_draft_workflow can reuse an injected SQLAlchemy session.""" - app = TestWorkflowAssociatedDataFactory.create_app() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() + # ==================== Get Published Workflow Tests ==================== + # These tests verify retrieval of published workflows (versioned snapshots) - result = workflow_service.get_draft_workflow(app, session=sqlite_session) + def test_get_published_workflow_by_id_success(self, workflow_service: WorkflowService, sqlite_session: Session): + """Test get_published_workflow_by_id returns published workflow.""" + app = TestWorkflowAssociatedDataFactory.create_app() + workflow_id = "workflow-123" + workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") + sqlite_session.add(workflow) + sqlite_session.commit() - assert result is workflow + result = workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) - def test_get_draft_workflow_returns_none(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test get_draft_workflow returns None when no draft exists.""" - app = TestWorkflowAssociatedDataFactory.create_app() + assert result is workflow - result = workflow_service.get_draft_workflow(app, session=sqlite_session) + def test_get_published_workflow_by_id_can_lock_restore_source( + self, workflow_service: WorkflowService, sqlite_session: Session, mocker: MockerFixture + ): + app = TestWorkflowAssociatedDataFactory.create_app() + workflow = TestWorkflowAssociatedDataFactory.create_workflow(version="v1") + session = sqlite_session + session.add(workflow) + session.commit() + lookup = mocker.spy(session, "scalar") - assert result is None + result = workflow_service.get_published_workflow_by_id( + app, + workflow.id, + session=session, + for_update=True, + ) - # ==================== Get Published Workflow Tests ==================== - # These tests verify retrieval of published workflows (versioned snapshots) + stmt = lookup.call_args.args[0] + sql = str(stmt.compile(dialect=postgresql.dialect())) + assert result is workflow + assert "FOR UPDATE" in sql - def test_get_published_workflow_by_id_reuses_provided_session(self, sqlite_session: Session): - """Published lookup sees uncommitted rows through the caller's session.""" - app = TestWorkflowAssociatedDataFactory.create_app() - workflow_id = "workflow-123" - workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") - sqlite_session.add(workflow) - sqlite_session.flush() + def test_get_published_workflow_by_id_raises_error_for_draft( + self, workflow_service: WorkflowService, sqlite_session: Session + ): + """ + Test get_published_workflow_by_id raises error when workflow is draft. - result = WorkflowService.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) + This prevents using draft workflows in production contexts where only + published, stable versions should be used (e.g., API execution). + """ + app = TestWorkflowAssociatedDataFactory.create_app() + workflow_id = "workflow-123" + workflow = TestWorkflowAssociatedDataFactory.create_workflow( + workflow_id=workflow_id, version=Workflow.VERSION_DRAFT + ) + sqlite_session.add(workflow) + sqlite_session.commit() - assert result is workflow + with pytest.raises(IsDraftWorkflowError): + workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) - def test_get_published_workflow_by_id_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test get_published_workflow_by_id returns published workflow.""" - app = TestWorkflowAssociatedDataFactory.create_app() - workflow_id = "workflow-123" - workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") - sqlite_session.add(workflow) - sqlite_session.commit() + def test_get_published_workflow_by_id_returns_none( + self, workflow_service: WorkflowService, sqlite_session: Session + ): + """Test get_published_workflow_by_id returns None when workflow not found.""" + app = TestWorkflowAssociatedDataFactory.create_app() + workflow_id = "nonexistent-workflow" - result = workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) + result = workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) - assert result is workflow + assert result is None - def test_get_published_workflow_by_id_can_lock_restore_source( - self, workflow_service: WorkflowService, sqlite_session: Session, mocker: MockerFixture - ): - app = TestWorkflowAssociatedDataFactory.create_app() - workflow = TestWorkflowAssociatedDataFactory.create_workflow(version="v1") - session = sqlite_session - session.add(workflow) - session.commit() - lookup = mocker.spy(session, "scalar") - - result = workflow_service.get_published_workflow_by_id( - app, - workflow.id, - session=session, - for_update=True, - ) + @pytest.mark.parametrize( + ("tenant_id", "app_id"), + [("other-tenant", "app-123"), ("tenant-456", "other-app")], + ) + def test_get_published_workflow_by_id_rejects_foreign_workflow( + self, + tenant_id: str, + app_id: str, + workflow_service: WorkflowService, + sqlite_session: Session, + ): + app = TestWorkflowAssociatedDataFactory.create_app() + workflow = TestWorkflowAssociatedDataFactory.create_workflow( + workflow_id="workflow-123", + tenant_id=tenant_id, + app_id=app_id, + version="v1", + ) + sqlite_session.add(workflow) + sqlite_session.commit() - stmt = lookup.call_args.args[0] - sql = str(stmt.compile(dialect=postgresql.dialect())) - assert result is workflow - assert "FOR UPDATE" in sql + result = workflow_service.get_published_workflow_by_id(app, workflow.id, session=sqlite_session) - def test_get_published_workflow_by_id_raises_error_for_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test get_published_workflow_by_id raises error when workflow is draft. - - This prevents using draft workflows in production contexts where only - published, stable versions should be used (e.g., API execution). - """ - app = TestWorkflowAssociatedDataFactory.create_app() - workflow_id = "workflow-123" - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, version=Workflow.VERSION_DRAFT - ) - sqlite_session.add(workflow) - sqlite_session.commit() + assert result is None - with pytest.raises(IsDraftWorkflowError): - workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) + def test_get_published_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): + """Test get_published_workflow returns published workflow.""" + workflow_id = "workflow-123" + app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=workflow_id) + workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") + sqlite_session.add(workflow) + sqlite_session.commit() - def test_get_published_workflow_by_id_returns_none( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test get_published_workflow_by_id returns None when workflow not found.""" - app = TestWorkflowAssociatedDataFactory.create_app() - workflow_id = "nonexistent-workflow" + result = workflow_service.get_published_workflow(app, session=sqlite_session) - result = workflow_service.get_published_workflow_by_id(app, workflow_id, session=sqlite_session) + assert result is workflow - assert result is None + def test_get_published_workflow_returns_none_when_no_workflow_id( + self, workflow_service: WorkflowService, sqlite_session: Session + ): + """Test get_published_workflow returns None when app has no workflow_id.""" + app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) - @pytest.mark.parametrize( - ("tenant_id", "app_id"), - [("other-tenant", "app-123"), ("tenant-456", "other-app")], - ) - def test_get_published_workflow_by_id_rejects_foreign_workflow( - self, - tenant_id: str, - app_id: str, - workflow_service: WorkflowService, - sqlite_session: Session, - ): - app = TestWorkflowAssociatedDataFactory.create_app() - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="workflow-123", - tenant_id=tenant_id, - app_id=app_id, - version="v1", - ) - sqlite_session.add(workflow) - sqlite_session.commit() + result = workflow_service.get_published_workflow(app, session=sqlite_session) - result = workflow_service.get_published_workflow_by_id(app, workflow.id, session=sqlite_session) + assert result is None - assert result is None + # ==================== Workflow Validation Tests ==================== + # These tests verify graph structure and feature configuration validation - def test_get_published_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test get_published_workflow returns published workflow.""" - workflow_id = "workflow-123" - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=workflow_id, version="v1") - sqlite_session.add(workflow) - sqlite_session.commit() + def test_validate_graph_structure_empty_graph(self, workflow_service: WorkflowService): + """Test validate_graph_structure accepts empty graph.""" + graph = {"nodes": []} - result = workflow_service.get_published_workflow(app, session=sqlite_session) + # Should not raise any exception + workflow_service.validate_graph_structure(graph) - assert result is workflow + def test_validate_graph_structure_valid_graph(self, workflow_service: WorkflowService): + """Test validate_graph_structure accepts valid graph.""" + graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - def test_get_published_workflow_returns_none_when_no_workflow_id( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test get_published_workflow returns None when app has no workflow_id.""" - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) + # Should not raise any exception + workflow_service.validate_graph_structure(graph) - result = workflow_service.get_published_workflow(app, session=sqlite_session) + def test_validate_graph_structure_start_and_trigger_coexist_raises_error(self, workflow_service: WorkflowService): + """ + Test validate_graph_structure raises error when start and trigger nodes coexist. - assert result is None + Workflows can be either: + - User-initiated (with START node): User provides input to start execution + - Event-driven (with trigger nodes): External events trigger execution - # ==================== Sync Draft Workflow Tests ==================== - # These tests verify creating and updating draft workflows with validation + These two patterns cannot be mixed in a single workflow. + """ + # Create a graph with both start and trigger nodes + # Use actual trigger node types: trigger-webhook, trigger-schedule, trigger-plugin + graph = { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "START", + }, + }, + { + "id": "trigger-1", + "data": { + "type": "trigger-webhook", + "title": "Webhook Trigger", + }, + }, + ], + "edges": [], + } - def test_sync_draft_workflow_creates_new_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test sync_draft_workflow creates new draft workflow when none exists. + with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): + workflow_service.validate_graph_structure(graph) - When a user first creates a workflow app, this creates the initial draft. - The draft is validated before creation to ensure graph and features are valid. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": False}} + def test_validate_features_structure_workflow_mode(self, workflow_service: WorkflowService): + """ + Test validate_features_structure for workflow mode. - with ( - patch("services.workflow_service.app_draft_workflow_was_synced"), - patch( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.sync_agent_bindings_for_draft", - return_value={"retired-agent"}, - ), - patch("services.workflow_service.WorkflowAgentRetirementService.retire_unowned") as retire_unowned, - ): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=None, - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) + Different app modes have different feature configurations. + This ensures the features match the expected schema for workflow apps. + """ + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) + features = {"file_upload": {"enabled": False}} - persisted_workflow = sqlite_session.scalar(select(Workflow).where(Workflow.id == result.id)) - assert persisted_workflow is result - assert result.graph_dict == graph - assert result.features_dict == features - retire_unowned.assert_called_once_with( - tenant_id=app.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, + with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True ) - def test_sync_draft_workflow_updates_existing_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test sync_draft_workflow updates existing draft workflow. - - When users edit their workflow, this updates the existing draft. - The unique_hash is used for optimistic locking to prevent conflicts. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": False}} - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=unique_hash, - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) + def test_validate_features_structure_advanced_chat_mode(self, workflow_service: WorkflowService): + """Test validate_features_structure for advanced chat mode.""" + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.ADVANCED_CHAT) + features = {"opening_statement": "Hello"} - sqlite_session.refresh(workflow) - assert result is workflow - assert workflow.graph_dict == graph - assert workflow.features_dict == features - assert workflow.updated_by == account.id - - def test_sync_draft_workflow_collaborative_save_preserves_environment_variables_and_locks_row( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A collaborative graph-only save locks the draft and keeps server environment values.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - remote_variable = StringVariable( - id="env-shared", - name="shared", - value="remote-value", - selector=["env", "shared"], - ) - stale_variable = remote_variable.model_copy(update={"value": "stale-client-value"}) - workflow.environment_variables = [remote_variable] - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - statements = [] - commits = [] - - @event.listens_for(sqlite_session, "do_orm_execute") - def capture_statement(execute_state): - statements.append(execute_state.statement) - - @event.listens_for(sqlite_session, "before_commit") - def capture_commit(_session): - commits.append(True) - - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=TestWorkflowAssociatedDataFactory.create_valid_workflow_graph(), - features={}, - unique_hash=unique_hash, - account=account, - environment_variables=[stale_variable], - conversation_variables=[], - session=sqlite_session, - preserve_environment_variables=True, - commit=False, - sync_agent_bindings=False, + with patch("services.workflow_service.AdvancedChatAppConfigManager.config_validate") as mock_validate: + workflow_service.validate_features_structure(app, features) + mock_validate.assert_called_once_with( + tenant_id=app.tenant_id, config=features, only_structure_validate=True ) - assert "FOR UPDATE" in str(statements[0].compile(dialect=postgresql.dialect())) - assert result.environment_variables == [remote_variable] - assert commits == [] - - def test_sync_draft_workflow_merges_environment_patch_with_graph_update( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """Graph changes and a per-ID environment patch commit without replacing untouched aliases.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - remote_variable = StringVariable(id="env-a", name="a", value="remote-a", selector=["env", "a"]) - replaced_variable = StringVariable(id="env-b", name="b", value="old-b", selector=["env", "b"]) - workflow.environment_variables = [remote_variable, replaced_variable] - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - next_graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - next_graph["viewport"] = {"x": 10, "y": 20, "zoom": 1} - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=next_graph, - features={}, - unique_hash=unique_hash, - account=account, - environment_variables=[ - remote_variable.model_copy(update={"value": "stale-a"}), - replaced_variable, - ], - conversation_variables=[], - session=sqlite_session, - environment_variable_upserts=[replaced_variable.model_copy(update={"value": "new-b"})], - deleted_environment_variable_ids=[], - preserve_environment_variables=True, - ) + def test_validate_features_structure_invalid_mode_raises_error(self, workflow_service: WorkflowService): + """Test validate_features_structure raises error for invalid mode.""" + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) + features = {} - assert result.graph_dict == next_graph - assert [(variable.id, variable.value) for variable in result.environment_variables] == [ - ("env-a", "remote-a"), - ("env-b", "new-b"), - ] + with pytest.raises(ValueError, match="Invalid app mode"): + workflow_service.validate_features_structure(app, features) - def test_sync_draft_workflow_noncollaborative_save_replaces_environment_variables( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """Legacy full-draft saves retain their full environment replacement contract.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - workflow.environment_variables = [ - StringVariable(id="env-old", name="old", value="old", selector=["env", "old"]) - ] - sqlite_session.add(workflow) - sqlite_session.commit() - replacement = StringVariable(id="env-new", name="new", value="new", selector=["env", "new"]) + # ==================== Publish Workflow Tests ==================== + # These tests verify creating published versions from draft workflows - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=TestWorkflowAssociatedDataFactory.create_valid_workflow_graph(), - features={}, - unique_hash=workflow.unique_hash, - account=account, - environment_variables=[replacement], - conversation_variables=[], - session=sqlite_session, - ) + # ==================== Version Management Tests ==================== + # These tests verify listing and managing published workflow versions - assert result.environment_variables == [replacement] + # ==================== Update Workflow Tests ==================== + # These tests verify updating workflow metadata (name, comments, etc.) - def test_sync_draft_workflow_graph_only_preserves_independently_updated_draft_fields( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - existing_features = {"file_upload": {"enabled": True}} - existing_environment_variables = [ - StringVariable(id="env-new", name="region", value="new", selector=["env", "region"]) - ] - existing_conversation_variables = [ - StringVariable( - id="conversation-new", - name="topic", - value="new", - selector=["conversation", "topic"], - ) - ] - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - features=existing_features, - environment_variables=existing_environment_variables, - conversation_variables=existing_conversation_variables, - ) - sqlite_session.add(workflow) - sqlite_session.commit() - unique_hash = workflow.unique_hash - updated_graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - updated_graph["nodes"][0]["data"]["title"] = "Updated graph" - - with patch("services.workflow_service.app_draft_workflow_was_synced"): - result = workflow_service.sync_draft_workflow( - app_model=app, - graph=updated_graph, - features={"file_upload": {"enabled": False}}, - unique_hash=unique_hash, - account=account, - environment_variables=[ - StringVariable(id="env-old", name="region", value="old", selector=["env", "region"]) - ], - conversation_variables=[ - StringVariable( - id="conversation-old", - name="topic", - value="old", - selector=["conversation", "topic"], - ) - ], - session=sqlite_session, - graph_only=True, - ) + # ==================== Get Default Block Config Tests ==================== + # These tests verify retrieval of default node configurations - sqlite_session.refresh(workflow) - assert result is workflow - assert workflow.graph_dict == updated_graph - assert workflow.features_dict == existing_features - assert workflow.environment_variables == existing_environment_variables - assert workflow.conversation_variables == existing_conversation_variables + def test_get_default_block_configs(self, workflow_service: WorkflowService): + """ + Test get_default_block_configs returns list of default configs. - def test_sync_draft_workflow_graph_only_creates_complete_initial_draft( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {"file_upload": {"enabled": True}} - environment_variables = [ - StringVariable(id="env-id", name="region", value="new", selector=["env", "region"]) - ] - conversation_variables = [ - StringVariable( - id="conversation-id", - name="topic", - value="new", - selector=["conversation", "topic"], - ) - ] + Returns default configurations for all available node types. + Used by the UI to populate the node palette and provide sensible defaults + when users add new nodes to their workflow. + """ + with patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping: + # Mock node class with default config + mock_node_class = MagicMock() + mock_node_class.get_default_config.return_value = {"type": "llm", "config": {}} - with patch("services.workflow_service.app_draft_workflow_was_synced"): - workflow = workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash=None, - account=account, - environment_variables=environment_variables, - conversation_variables=conversation_variables, - session=sqlite_session, - graph_only=True, - ) + mock_mapping.return_value = {BuiltinNodeTypes.LLM: {"latest": mock_node_class}} - assert workflow.graph_dict == graph - assert workflow.features_dict == features - assert workflow.environment_variables == environment_variables - assert workflow.conversation_variables == conversation_variables + with patch("services.workflow_service.LATEST_VERSION", "latest"): + result = workflow_service.get_default_block_configs() - def test_sync_draft_workflow_raises_hash_not_equal_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test sync_draft_workflow raises error when hash doesn't match. - - This implements optimistic locking: if the workflow was modified by another - user/session since it was loaded, the hash won't match and the update fails. - This prevents overwriting concurrent changes. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - features = {} - - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() - - with pytest.raises(WorkflowHashNotEqualError): - workflow_service.sync_draft_workflow( - app_model=app, - graph=graph, - features=features, - unique_hash="new-hash", - account=account, - environment_variables=[], - conversation_variables=[], - session=sqlite_session, - ) + assert len(result) > 0 + + def test_get_default_block_configs_http_request_injects_default_config(self, workflow_service: WorkflowService): + injected_config = HttpRequestNodeConfig( + max_connect_timeout=15, + max_read_timeout=25, + max_write_timeout=35, + max_binary_size=4096, + max_text_size=2048, + ssl_verify=True, + ssrf_default_max_retries=6, + ) - def test_restore_published_workflow_to_draft_keeps_source_features_unmodified( - self, workflow_service: WorkflowService, sqlite_session: Session + with ( + patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, + patch("services.workflow_service.LATEST_VERSION", "latest"), + patch( + "services.workflow_service.build_http_request_config", + return_value=injected_config, + ) as mock_build_config, ): - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - legacy_features = { - "file_upload": { - "image": { - "enabled": True, - "number_limits": 6, - "transfer_methods": ["remote_url", "local_file"], - } - }, - "opening_statement": "", - "retriever_resource": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": False}, - "speech_to_text": {"enabled": False}, - "suggested_questions": [], - "suggested_questions_after_answer": {"enabled": False}, - "text_to_speech": {"enabled": False, "language": "", "voice": ""}, + mock_http_node_class = MagicMock() + mock_http_node_class.get_default_config.return_value = {"type": "http-request", "config": {}} + mock_llm_node_class = MagicMock() + mock_llm_node_class.get_default_config.return_value = {"type": "llm", "config": {}} + mock_mapping.return_value = { + BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_http_node_class}, + BuiltinNodeTypes.LLM: {"latest": mock_llm_node_class}, } - source_workflow = Workflow( - id="published-workflow-id", - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version="2026-03-19T00:00:00", - graph=json.dumps(TestWorkflowAssociatedDataFactory.create_valid_workflow_graph()), - _features=json.dumps(legacy_features), - created_by=account.id, - _environment_variables=[], - _conversation_variables=[], - _rag_pipeline_variables=[], - ) - draft_workflow = Workflow( - id="draft-workflow-id", - tenant_id=app.tenant_id, - app_id=app.id, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps({"nodes": [], "edges": []}), - _features=json.dumps({}), - created_by=account.id, - _environment_variables=[], - _conversation_variables=[], - _rag_pipeline_variables=[], - ) - sqlite_session.add_all([source_workflow, draft_workflow]) - sqlite_session.commit() - with ( - patch("services.workflow_service.app_draft_workflow_was_synced"), - patch.object( - workflow_service, - "get_published_workflow_by_id", - wraps=workflow_service.get_published_workflow_by_id, - ) as get_published_workflow_by_id, - patch( - "services.agent.legacy_workflow_publish_service.WorkflowAgentPublishService.restore_agent_node_bindings_to_draft", - return_value={"retired-agent"}, - ), - patch("services.workflow_service.WorkflowAgentRetirementService.retire_unowned") as retire_unowned, - ): - result = workflow_service.restore_published_workflow_to_draft( - app_model=app, - workflow_id=source_workflow.id, - account=account, - session=sqlite_session, - ) + result = workflow_service.get_default_block_configs() - assert result is draft_workflow - assert source_workflow.serialized_features == json.dumps(legacy_features) - assert draft_workflow.serialized_features == json.dumps(legacy_features) - sqlite_session.refresh(draft_workflow) - assert draft_workflow.serialized_features == json.dumps(legacy_features) - get_published_workflow_by_id.assert_called_once_with( - app_model=app, - workflow_id=source_workflow.id, - session=sqlite_session, - for_update=True, - ) - retire_unowned.assert_called_once_with( - tenant_id=app.tenant_id, - agent_ids={"retired-agent"}, - account_id=account.id, - ) + assert result == [ + {"type": "http-request", "config": {}}, + {"type": "llm", "config": {}}, + ] + mock_build_config.assert_called_once() + passed_http_filters = mock_http_node_class.get_default_config.call_args.kwargs["filters"] + assert passed_http_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is injected_config + mock_llm_node_class.get_default_config.assert_called_once_with(filters=None) - def test_restore_historical_inline_agent_after_current_pointer_moves_uses_real_clone( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) - account = TestWorkflowAssociatedDataFactory.create_account() - graph = { - "nodes": [ - { - "id": "agent-node", - "data": { - "type": "agent", - "version": "2", - "agent_node_kind": "dify_agent", - }, - } - ], - "edges": [], - } - historical = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="historical-workflow", - version="historical-version", - graph=graph, - ) - current = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="current-workflow", - version="current-version", - graph=graph, - ) - draft = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="draft-workflow", - version=Workflow.VERSION_DRAFT, - ) - source_agent = Agent( - id="historical-agent", - tenant_id=app.tenant_id, - name="Historical inline Agent", - description="", - role="", - agent_kind=AgentKind.DIFY_AGENT, - scope=AgentScope.WORKFLOW_ONLY, - source=AgentSource.WORKFLOW, - app_id=app.id, - workflow_id=historical.id, - workflow_node_id="agent-node", - active_config_snapshot_id="historical-snapshot", - active_config_has_model=False, - active_config_is_published=True, - status=AgentStatus.ACTIVE, - created_by=account.id, - updated_by=account.id, - ) - source_snapshot = AgentConfigSnapshot( - id="historical-snapshot", - tenant_id=app.tenant_id, - agent_id=source_agent.id, - version=1, - config_snapshot=AgentSoulConfig(config_note="historical soul"), - created_by=account.id, - ) - historical_binding = WorkflowAgentNodeBinding( - id="historical-binding", - tenant_id=app.tenant_id, - app_id=app.id, - workflow_id=historical.id, - workflow_version=historical.version, - node_id="agent-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id=source_agent.id, - current_snapshot_id=source_snapshot.id, - node_job_config={}, - created_by=account.id, - ) - sqlite_session.add_all([app, historical, current, draft, source_agent, source_snapshot, historical_binding]) - app.workflow_id = historical.id - sqlite_session.commit() + def test_get_default_block_config_for_node_type(self, workflow_service: WorkflowService): + """ + Test get_default_block_config returns config for specific node type. - app.workflow_id = current.id - sqlite_session.commit() + Returns the default configuration for a specific node type (e.g., LLM, HTTP). + This includes default values for all required and optional parameters. + """ + with ( + patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, + patch("services.workflow_service.LATEST_VERSION", "latest"), + ): + # Mock node class with default config + mock_node_class = MagicMock() + mock_config = {"type": "llm", "config": {"provider": "openai"}} + mock_node_class.get_default_config.return_value = mock_config + + # Create a mock mapping that includes BuiltinNodeTypes.LLM + mock_mapping.return_value = {BuiltinNodeTypes.LLM: {"latest": mock_node_class}} + + result = workflow_service.get_default_block_config(BuiltinNodeTypes.LLM) + + assert result == mock_config + mock_node_class.get_default_config.assert_called_once() + + def test_get_default_block_config_invalid_node_type(self, workflow_service: WorkflowService): + """Test get_default_block_config returns empty dict for invalid node type.""" + with patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping: + mock_mapping.return_value = {} + + # Use a valid NodeType but one that's not in the mapping + result = workflow_service.get_default_block_config(BuiltinNodeTypes.LLM) + + assert result == {} + + def test_get_default_block_config_http_request_injects_default_config(self, workflow_service: WorkflowService): + injected_config = HttpRequestNodeConfig( + max_connect_timeout=11, + max_read_timeout=22, + max_write_timeout=33, + max_binary_size=4096, + max_text_size=2048, + ssl_verify=False, + ssrf_default_max_retries=7, + ) - WorkflowAgentRetirementService.retire_unowned( - tenant_id=app.tenant_id, - agent_ids=[source_agent.id], - account_id=account.id, - ) - sqlite_session.expire_all() - retained_agent = sqlite_session.get(Agent, source_agent.id) - assert retained_agent is not None - assert retained_agent.status is AgentStatus.ACTIVE + with ( + patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, + patch("services.workflow_service.LATEST_VERSION", "latest"), + patch( + "services.workflow_service.build_http_request_config", + return_value=injected_config, + ) as mock_build_config, + ): + mock_node_class = MagicMock() + expected = {"type": "http-request", "config": {}} + mock_node_class.get_default_config.return_value = expected + mock_mapping.return_value = {BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_node_class}} + + result = workflow_service.get_default_block_config(BuiltinNodeTypes.HTTP_REQUEST) + + assert result == expected + mock_build_config.assert_called_once() + passed_filters = mock_node_class.get_default_config.call_args.kwargs["filters"] + assert passed_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is injected_config + + def test_get_default_block_config_http_request_uses_passed_config(self, workflow_service: WorkflowService): + provided_config = HttpRequestNodeConfig( + max_connect_timeout=13, + max_read_timeout=23, + max_write_timeout=34, + max_binary_size=8192, + max_text_size=4096, + ssl_verify=True, + ssrf_default_max_retries=2, + ) - with patch("services.workflow_service.app_draft_workflow_was_synced"): - restored_draft = workflow_service.restore_published_workflow_to_draft( - app_model=app, - workflow_id=historical.id, - account=account, - session=sqlite_session, - ) + with ( + patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, + patch("services.workflow_service.LATEST_VERSION", "latest"), + patch("services.workflow_service.build_http_request_config") as mock_build_config, + ): + mock_node_class = MagicMock() + expected = {"type": "http-request", "config": {}} + mock_node_class.get_default_config.return_value = expected + mock_mapping.return_value = {BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_node_class}} - restored_binding = sqlite_session.scalar( - select(WorkflowAgentNodeBinding).where( - WorkflowAgentNodeBinding.workflow_id == draft.id, - WorkflowAgentNodeBinding.workflow_version == Workflow.VERSION_DRAFT, - WorkflowAgentNodeBinding.node_id == "agent-node", - ) + result = workflow_service.get_default_block_config( + BuiltinNodeTypes.HTTP_REQUEST, + filters={HTTP_REQUEST_CONFIG_FILTER_KEY: provided_config}, ) - assert restored_draft is draft - assert app.workflow_id == current.id - assert sqlite_session.get(Workflow, historical.id) is historical - assert sqlite_session.get(WorkflowAgentNodeBinding, historical_binding.id) is historical_binding - assert restored_binding is not None - assert restored_binding.agent_id not in (None, source_agent.id) - assert restored_binding.current_snapshot_id not in (None, source_snapshot.id) - restored_agent = sqlite_session.get(Agent, restored_binding.agent_id) - restored_snapshot = sqlite_session.get(AgentConfigSnapshot, restored_binding.current_snapshot_id) - assert restored_agent is not None - assert restored_agent.workflow_id == draft.id - assert restored_agent.workflow_node_id == "agent-node" - assert restored_snapshot is not None - assert restored_snapshot.config_snapshot_dict == source_snapshot.config_snapshot_dict - - # ==================== Workflow Validation Tests ==================== - # These tests verify graph structure and feature configuration validation - - def test_validate_graph_structure_empty_graph(self, workflow_service: WorkflowService): - """Test validate_graph_structure accepts empty graph.""" - graph = {"nodes": []} - - # Should not raise any exception - workflow_service.validate_graph_structure(graph) - def test_validate_graph_structure_valid_graph(self, workflow_service: WorkflowService): - """Test validate_graph_structure accepts valid graph.""" - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() + assert result == expected + mock_build_config.assert_not_called() + passed_filters = mock_node_class.get_default_config.call_args.kwargs["filters"] + assert passed_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is provided_config - # Should not raise any exception - workflow_service.validate_graph_structure(graph) - - def test_validate_graph_structure_start_and_trigger_coexist_raises_error( - self, workflow_service: WorkflowService + def test_get_default_block_config_http_request_malformed_config_raises_type_error( + self, workflow_service: WorkflowService + ): + with ( + patch( + "services.workflow_service.get_node_type_classes_mapping", + return_value={BuiltinNodeTypes.HTTP_REQUEST: {"latest": HttpRequestNode}}, + ), + patch("services.workflow_service.LATEST_VERSION", "latest"), ): - """ - Test validate_graph_structure raises error when start and trigger nodes coexist. - - Workflows can be either: - - User-initiated (with START node): User provides input to start execution - - Event-driven (with trigger nodes): External events trigger execution - - These two patterns cannot be mixed in a single workflow. - """ - # Create a graph with both start and trigger nodes - # Use actual trigger node types: trigger-webhook, trigger-schedule, trigger-plugin - graph = { - "nodes": [ - { - "id": "start", - "data": { - "type": "start", - "title": "START", - }, - }, - { - "id": "trigger-1", - "data": { - "type": "trigger-webhook", - "title": "Webhook Trigger", - }, - }, - ], - "edges": [], - } + with pytest.raises(TypeError, match="http_request_config must be an HttpRequestNodeConfig instance"): + workflow_service.get_default_block_config( + BuiltinNodeTypes.HTTP_REQUEST, + filters={HTTP_REQUEST_CONFIG_FILTER_KEY: "invalid"}, + ) - with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): - workflow_service.validate_graph_structure(graph) - def test_validate_features_structure_workflow_mode(self, workflow_service: WorkflowService): - """ - Test validate_features_structure for workflow mode. +# =========================================================================== +# TestWorkflowServiceCredentialValidation +# Tests for _validate_workflow_credentials and related private helpers +# =========================================================================== - Different app modes have different feature configurations. - This ensures the features match the expected schema for workflow apps. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - features = {"file_upload": {"enabled": False}} - with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_validate: - workflow_service.validate_features_structure(app, features) - mock_validate.assert_called_once_with( - tenant_id=app.tenant_id, config=features, only_structure_validate=True - ) +@pytest.mark.usefixtures("sqlite_session") +class TestWorkflowServiceCredentialValidation: + """ + Tests for the private credential-validation helpers on WorkflowService. - def test_validate_features_structure_advanced_chat_mode(self, workflow_service: WorkflowService): - """Test validate_features_structure for advanced chat mode.""" - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.ADVANCED_CHAT) - features = {"opening_statement": "Hello"} + These helpers gate `publish_workflow` when `PluginManager` is enabled. + Each test focuses on a distinct branch inside `_validate_workflow_credentials`, + `_validate_llm_model_config`, `_check_default_tool_credential`, and the + load-balancing path. + """ - with patch("services.workflow_service.AdvancedChatAppConfigManager.config_validate") as mock_validate: - workflow_service.validate_features_structure(app, features) - mock_validate.assert_called_once_with( - tenant_id=app.tenant_id, config=features, only_structure_validate=True - ) + @pytest.fixture + def service(self, sqlite_engine: Engine) -> WorkflowService: + return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - def test_validate_features_structure_invalid_mode_raises_error(self, workflow_service: WorkflowService): - """Test validate_features_structure raises error for invalid mode.""" - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) - features = {} + @pytest.fixture(autouse=True) + def credential_repository(self, sqlite_session_factory: sessionmaker[Session]) -> None: + self.credentials = CredentialQueryRepository(session_factory=sqlite_session_factory) - with pytest.raises(ValueError, match="Invalid app mode"): - workflow_service.validate_features_structure(app, features) + @staticmethod + def _make_workflow(nodes: list[dict]) -> Workflow: + return TestWorkflowAssociatedDataFactory.create_workflow( + tenant_id="tenant-1", + app_id="app-1", + graph={"nodes": nodes, "edges": []}, + ) - # ==================== Draft Workflow Variable Update Tests ==================== - # These tests verify updating draft workflow environment/conversation variables + def test_validate_publication_checks_knowledge_fs_control_spaces( + self, + service: WorkflowService, + sqlite_engine: Engine, + config_overrides: Callable[..., None], + ) -> None: + config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) + app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") + workflow = self._make_workflow([]) - def test_update_draft_workflow_environment_variables_updates_workflow( - self, workflow_service: WorkflowService, sqlite_session: Session + with ( + patch("services.workflow_service.SystemFeatureService.is_plugin_manager_enabled", return_value=False), + patch.object(service, "validate_graph_structure"), + patch( + "services.knowledge_fs.knowledge_retrieval_v2.validation.validate_control_space_references" + ) as validate_control_spaces, ): - """Test update_draft_workflow_environment_variables updates draft fields.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - variables = [StringVariable(id="env-id", name="region", value="us-east-1", selector=["env", "region"])] - - workflow_service.update_draft_workflow_environment_variables( - app_model=app, - environment_variables=variables, - account=account, - session=sqlite_session, - ) + service.validate_publication(app, workflow, credentials=self.credentials) - assert workflow.environment_variables == variables - assert workflow.updated_by == account.id - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert persisted_workflow.environment_variables == variables + validation = validate_control_spaces.call_args.kwargs + assert validation["session"].get_bind() is sqlite_engine + assert not validation["session"].in_transaction() + assert validation["tenant_id"] == app.tenant_id + assert validation["graph"] == workflow.graph_dict - def test_update_draft_workflow_environment_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_draft_workflow_environment_variables raises when draft missing.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() + # --- _validate_workflow_credentials: tool node (with credential_id) --- - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.update_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - account=account, - session=sqlite_session, - ) + def test_validate_workflow_credentials_should_check_tool_credential_when_credential_id_present( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + nodes = [ + { + "id": "tool-node", + "data": { + "type": "tool", + "provider_id": "my-provider", + "credential_id": "cred-123", + }, + } + ] + workflow = self._make_workflow(nodes) - def test_patch_draft_workflow_environment_variables_merges_by_id( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch preserves untouched variables while applying ordered upserts and deletions.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - workflow.environment_variables = [ - StringVariable(id="env-a", name="a", value="remote-a", selector=["env", "a"]), - StringVariable(id="env-b", name="b", value="old-b", selector=["env", "b"]), - StringVariable(id="env-c", name="c", value="remote-c", selector=["env", "c"]), - ] - sqlite_session.add(workflow) - sqlite_session.commit() - - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[ - StringVariable(id="env-b", name="b", value="new-b", selector=["env", "b"]), - StringVariable(id="env-d", name="d", value="new-d", selector=["env", "d"]), - ], - deleted_environment_variable_ids=["env-a"], - account=account, - session=sqlite_session, - ) + # Act + Assert + with patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check: + # Should not raise; mock allows the call + service._validate_workflow_credentials(workflow, credentials=self.credentials) + mock_check.assert_called_once() - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert [(variable.id, variable.value) for variable in persisted_workflow.environment_variables] == [ - ("env-b", "new-b"), - ("env-c", "remote-c"), - ("env-d", "new-d"), - ] - assert persisted_workflow.updated_by == account.id - - def test_patch_draft_workflow_environment_variables_locks_draft_row( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """The merge reads the draft with a row lock before applying a partial update.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - statements = [] - - @event.listens_for(sqlite_session, "do_orm_execute") - def capture_statement(execute_state): - statements.append(execute_state.statement) - - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - deleted_environment_variable_ids=[], - account=account, - session=sqlite_session, - ) + def test_validate_workflow_credentials_should_check_default_credential_when_no_credential_id( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + nodes = [ + { + "id": "tool-node", + "data": { + "type": "tool", + "provider_id": "my-provider", + # No credential_id — should fall back to default + }, + } + ] + workflow = self._make_workflow(nodes) - compiled_statement = str(statements[0].compile(dialect=postgresql.dialect())) - assert "FOR UPDATE" in compiled_statement - assert sqlite_session.get(Workflow, workflow.id) is workflow - - def test_patch_draft_workflow_environment_variables_rejects_conflicting_ids( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch cannot both upsert and delete the same variable ID.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - variable = StringVariable(id="env-a", name="a", value="a", selector=["env", "a"]) - sqlite_session.add(TestWorkflowAssociatedDataFactory.create_workflow()) - sqlite_session.commit() - - with pytest.raises(ValueError, match="cannot be upserted and deleted"): - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[variable], - deleted_environment_variable_ids=["env-a"], - account=account, - session=sqlite_session, - ) + # Act + with patch.object(service, "_check_default_tool_credential") as mock_default: + service._validate_workflow_credentials(workflow, credentials=self.credentials) - def test_patch_draft_workflow_environment_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session - ) -> None: - """A patch fails when the app has no draft workflow to lock.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() + # Assert + mock_default.assert_called_once_with("tenant-1", "my-provider", credentials=self.credentials) - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.patch_draft_workflow_environment_variables( - app_model=app, - environment_variables=[], - deleted_environment_variable_ids=[], - account=account, - session=sqlite_session, - ) + def test_validate_workflow_credentials_should_skip_tool_node_without_provider( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """Tool nodes without a provider_id should be silently skipped.""" + # Arrange + nodes = [{"id": "tool-node", "data": {"type": "tool"}}] + workflow = self._make_workflow(nodes) + + # Act + Assert (no error raised) + with patch.object(service, "_check_default_tool_credential") as mock_default: + service._validate_workflow_credentials(workflow, credentials=self.credentials) + mock_default.assert_not_called() + + def test_validate_workflow_credentials_should_validate_llm_node_with_model_config( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + nodes = [ + { + "id": "llm-node", + "data": { + "type": "llm", + "model": {"provider": "openai", "name": "gpt-4"}, + }, + } + ] + workflow = self._make_workflow(nodes) - def test_update_draft_workflow_conversation_variables_updates_workflow( - self, workflow_service: WorkflowService, sqlite_session: Session + # Act + with ( + patch.object(service, "_validate_llm_model_config") as mock_llm, + patch.object(service, "_validate_load_balancing_credentials"), ): - """Test update_draft_workflow_conversation_variables updates draft fields.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - sqlite_session.add(workflow) - sqlite_session.commit() - variables = [ - StringVariable(id="conversation-id", name="topic", value="sqlite", selector=["conversation", "topic"]) - ] + service._validate_workflow_credentials(workflow, credentials=self.credentials) - workflow_service.update_draft_workflow_conversation_variables( - app_model=app, - conversation_variables=variables, - account=account, - session=sqlite_session, - ) + # Assert + mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") - assert workflow.conversation_variables == variables - assert workflow.updated_by == account.id - sqlite_session.expire_all() - persisted_workflow = sqlite_session.get(Workflow, workflow.id) - assert persisted_workflow is not None - assert persisted_workflow.conversation_variables == variables + def test_validate_workflow_credentials_should_use_llm_environment_variable_model( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + workflow = self._make_workflow( + [ + { + "id": "llm-node", + "data": { + "type": "llm", + "model": { + "provider": "old-provider", + "name": "old-model", + "mode": "chat", + "completion_params": {"temperature": 0.2}, + }, + "model_selector": ["env", "shared_model"], + }, + } + ] + ) + workflow.environment_variables = [ + LLMEnvironmentVariable( + name="shared_model", + value={ + "provider": "new-provider", + "name": "new-model", + "mode": "chat", + "completion_params": {"temperature": 0.8}, + }, + ) + ] - def test_update_draft_workflow_conversation_variables_raises_when_missing( - self, workflow_service: WorkflowService, sqlite_session: Session + with ( + patch.object(service, "_validate_llm_model_config") as validate_model, + patch.object(service, "_validate_load_balancing_credentials") as validate_load_balancing, ): - """Test update_draft_workflow_conversation_variables raises when draft missing.""" - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No draft workflow found."): - workflow_service.update_draft_workflow_conversation_variables( - app_model=app, - conversation_variables=[], - account=account, - session=sqlite_session, - ) - - # ==================== Publish Workflow Tests ==================== - # These tests verify creating published versions from draft workflows - - def test_publish_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test publish_workflow creates new published version. - - Publishing creates a timestamped snapshot of the draft workflow. - This allows users to: - - Roll back to previous versions - - Use stable versions in production - - Continue editing draft without affecting published version - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with patch("services.workflow_service.app_published_workflow_was_updated"): - result = workflow_service.publish_workflow( - session=sqlite_session, - app_model=app, - account=account, - marked_name="Version 1", - marked_comment="Initial release", - ) - - sqlite_session.flush() - assert result in sqlite_session - assert result.version != Workflow.VERSION_DRAFT - assert result.marked_name == "Version 1" - assert result.marked_comment == "Initial release" - - def test_publish_workflow_numbers_versions_from_one( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test publish_workflow assigns an app-scoped version number starting at #1. - - The number is what users see when a version carries no name, so it has to be - stable and monotonic per app rather than derived from list position. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - first = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - second = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - assert first.version_number == 1 - assert second.version_number == 2 - - def test_publish_workflow_does_not_reuse_a_deleted_version_number( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test version numbers are never handed out twice, even after a version is deleted. - - Deployment records and audit logs refer to versions by number, so reusing one - would make two different workflows share an identity. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - published = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - sqlite_session.flush() - sqlite_session.delete(published) - sqlite_session.flush() - - republished = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - assert republished.version_number == 2 - - def test_publish_workflow_numbers_each_app_independently( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test the counter is scoped per app rather than global. - - Every app starts its own sequence at #1; a busy neighbour must not advance it. - """ - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - - published: list[Workflow] = [] - for app_id in ("app-first", "app-second"): - app = TestWorkflowAssociatedDataFactory.create_app(app_id=app_id) - draft = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=f"draft-{app_id}", - app_id=app_id, - version=Workflow.VERSION_DRAFT, - graph=graph, - ) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.app_published_workflow_was_updated"), - ): - workflow = workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - published.append(workflow) - - assert [workflow.version_number for workflow in published] == [1, 1] - - def test_publish_workflow_no_draft_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test publish_workflow raises error when no draft exists. - - Cannot publish if there's no draft to publish from. - Users must create and save a draft before publishing. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - with pytest.raises(ValueError, match="No valid workflow found"): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - @pytest.mark.parametrize( - ("environment_variables", "error"), + service._validate_workflow_credentials(workflow, credentials=self.credentials) + + validate_model.assert_called_once_with("tenant-1", "new-provider", "new-model") + validated_node_data = validate_load_balancing.call_args.args[1] + assert validated_node_data["model"] == { + "provider": "new-provider", + "name": "new-model", + "mode": "chat", + "completion_params": {"temperature": 0.8}, + } + + def test_validate_workflow_credentials_should_reject_llm_environment_variable_mode_mismatch( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + workflow = self._make_workflow( [ - ([], "was not found or is not an LLM variable"), - ( - [ - LLMEnvironmentVariable( - name="shared_model", - value={"provider": "provider", "name": "model", "mode": "completion"}, - ) - ], - "uses mode 'completion'.*uses mode 'chat'", - ), - ], + { + "id": "llm-node", + "data": { + "type": "llm", + "model": {"provider": "provider", "name": "model", "mode": "chat"}, + "model_selector": ["env", "shared_model"], + }, + } + ] ) - def test_publish_workflow_rejects_invalid_llm_environment_reference_without_credential_validation( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - environment_variables: list[LLMEnvironmentVariable], - error: str, - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - graph = TestWorkflowAssociatedDataFactory.create_valid_workflow_graph() - llm_node = next(node for node in graph["nodes"] if node["data"]["type"] == BuiltinNodeTypes.LLM) - llm_node["data"]["model"] = { - "provider": "provider", - "name": "model", - "mode": "chat", - "completion_params": {}, - } - llm_node["data"]["model_selector"] = ["env", "shared_model"] - draft = TestWorkflowAssociatedDataFactory.create_workflow(graph=graph) - draft.environment_variables = environment_variables - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch( - "services.system_feature_service.SystemFeatureService.is_plugin_manager_enabled", - return_value=False, - ), - pytest.raises(ValueError, match=error), - ): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - def test_publish_workflow_trigger_limit_exceeded( - self, - workflow_service: WorkflowService, - sqlite_session: Session, - config_overrides: Callable[..., None], - ): - config_overrides(DEPLOYMENT_EDITION=DeploymentEdition.CLOUD) - """ - Test publish_workflow raises error when trigger node limit exceeded in SANDBOX plan. - - Free/sandbox tier users have limits on the number of trigger nodes. - This prevents resource abuse while allowing users to test the feature. - The limit is enforced at publish time, not during draft editing. - """ - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - - # Create graph with 3 trigger nodes (exceeds SANDBOX limit of 2) - # Trigger nodes enable event-driven automation which consumes resources - graph = { - "nodes": [ - {"id": "trigger-1", "data": {"type": "trigger-webhook"}}, - {"id": "trigger-2", "data": {"type": "trigger-schedule"}}, - {"id": "trigger-3", "data": {"type": "trigger-plugin"}}, - ], - "edges": [], - } - draft = TestWorkflowAssociatedDataFactory.create_workflow(version=Workflow.VERSION_DRAFT, graph=graph) - sqlite_session.add(draft) - sqlite_session.commit() - - with ( - patch("services.workflow_service.BillingService") as mock_billing_service, - ): - mock_billing_service.get_info.return_value = builders.subscription_info("sandbox") - - with pytest.raises(TriggerNodeLimitExceededError): - workflow_service.publish_workflow(session=sqlite_session, app_model=app, account=account) - - # ==================== Version Management Tests ==================== - # These tests verify listing and managing published workflow versions - - def test_get_all_published_workflow_with_pagination( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test get_all_published_workflow returns paginated results. - - Apps can have many published versions over time. - Pagination prevents loading all versions at once, improving performance. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-123") - - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=f"workflow-{i}", version=f"v{i}") - for i in range(5) - ] - ) - sqlite_session.commit() - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert len(workflows) == 5 - assert has_more is False - - def test_get_all_published_workflow_lists_the_draft_first( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test the draft heads the version list no matter how old it is. - - A draft is created together with its app and its `created_at` is never refreshed, - so ordering purely by publish time would put it last — off the first page entirely - once the app has accumulated enough published versions. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-3") - app_created_at = datetime(2026, 1, 1) - - sqlite_session.add( - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="workflow-draft", - version=Workflow.VERSION_DRAFT, - created_at=app_created_at, - ) - ) - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=f"workflow-{i}", - version=f"2026-02-0{i} 00:00:00", - created_at=app_created_at + timedelta(days=i), - ) - for i in range(1, 4) - ] - ) - sqlite_session.commit() - - workflows, _ = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=2, user_id=None - ) - - assert [workflow.id for workflow in workflows] == ["workflow-draft", "workflow-3"] - - def test_get_all_published_workflow_excludes_draft_when_requested( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """`include_draft=False` filters the draft out at query level so callers that - only understand published versions (e.g. the openapi version-list op, where the - draft breaks restore and ordering) get an exact page. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-3") - app_created_at = datetime(2026, 1, 1) - - sqlite_session.add( - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="workflow-draft", - version=Workflow.VERSION_DRAFT, - created_at=app_created_at, - ) - ) - sqlite_session.add( - TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="workflow-3", - version="2026-02-01 00:00:00", - created_at=app_created_at + timedelta(days=1), - ) - ) - sqlite_session.commit() - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None, include_draft=False - ) - - assert [workflow.id for workflow in workflows] == ["workflow-3"] - assert has_more is False - - def test_get_all_published_workflow_has_more(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test get_all_published_workflow indicates has_more when results exceed limit. - - The has_more flag tells the UI whether to show a "Load More" button. - This is determined by fetching limit+1 records and checking if we got that many. - """ - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id="workflow-123") - - sqlite_session.add_all( - [ - TestWorkflowAssociatedDataFactory.create_workflow(workflow_id=f"workflow-{i}", version=f"v{i}") - for i in range(11) - ] - ) - sqlite_session.commit() - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert len(workflows) == 10 - assert has_more is True - - def test_get_all_published_workflow_no_workflow_id( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test get_all_published_workflow returns empty when app has no workflow_id.""" - app = TestWorkflowAssociatedDataFactory.create_app(workflow_id=None) - - workflows, has_more = workflow_service.get_all_published_workflow( - session=sqlite_session, app_model=app, page=1, limit=10, user_id=None - ) - - assert workflows == [] - assert has_more is False - - # ==================== Update Workflow Tests ==================== - # These tests verify updating workflow metadata (name, comments, etc.) - - def test_update_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test update_workflow updates workflow attributes. - - Allows updating metadata like marked_name and marked_comment - without creating a new version. Only specific fields are allowed - to prevent accidental modification of workflow logic. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id=app_id, workflow_id=workflow_id) - account_id = "user-123" - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - result = workflow_service.update_workflow( - session=sqlite_session, - account_id=account_id, - data={"marked_name": "Updated Name", "marked_comment": "Updated Comment"}, - workflow_ref=workflow_ref, - ) - - sqlite_session.flush() - assert result is workflow - assert workflow.marked_name == "Updated Name" - assert workflow.marked_comment == "Updated Comment" - assert workflow.updated_by == account_id - - def test_update_workflow_not_found(self, workflow_service: WorkflowService, sqlite_session: Session): - """Test update_workflow returns None when workflow not found.""" - result = workflow_service.update_workflow( - session=sqlite_session, - account_id="user-123", - data={"marked_name": "Test"}, - workflow_ref=WorkflowRef(tenant_id="tenant-456", owner_id="app-789", workflow_id="nonexistent"), + workflow.environment_variables = [ + LLMEnvironmentVariable( + name="shared_model", + value={"provider": "provider", "name": "model", "mode": "completion"}, ) + ] - assert result is None + with pytest.raises(ValueError, match="uses mode 'completion'.*uses mode 'chat'"): + service._validate_workflow_credentials(workflow, credentials=self.credentials) - def test_update_workflow_with_ref_scopes_lookup_to_app( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test update_workflow includes the trusted app owner in the lookup.""" - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - account_id = "user-123" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="other-app", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - result = workflow_service.update_workflow( - session=sqlite_session, - account_id=account_id, - data={"marked_name": "Updated Name"}, - workflow_ref=workflow_ref, - ) - - assert result is None - assert workflow.marked_name == "" - - # ==================== Delete Workflow Tests ==================== - # These tests verify workflow deletion with safety checks - - def test_delete_workflow_success(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test delete_workflow successfully deletes a published workflow. - - Users can delete old published versions they no longer need. - This helps manage storage and keeps the version list clean. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id=app_id, workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id, version="v1" - ) - inline_binding = WorkflowAgentNodeBinding( - id="inline-binding", - tenant_id=tenant_id, - app_id=app_id, - workflow_id=workflow_id, - workflow_version="v1", - node_id="inline-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id="inline-agent", - current_snapshot_id="snapshot-1", - node_job_config={}, - ) - roster_binding = WorkflowAgentNodeBinding( - id="roster-binding", - tenant_id=tenant_id, - app_id=app_id, - workflow_id=workflow_id, - workflow_version="v1", - node_id="roster-node", - binding_type=WorkflowAgentBindingType.ROSTER_AGENT, - agent_id="roster-agent", - current_snapshot_id="snapshot-2", - node_job_config={}, - ) - non_target_bindings = [ - WorkflowAgentNodeBinding( - id=f"non-target-{key}", - tenant_id="other-tenant" if key == "tenant" else tenant_id, - app_id="other-app" if key == "app" else app_id, - workflow_id="other-workflow" if key == "workflow" else workflow_id, - workflow_version="other-version" if key == "version" else workflow.version, - node_id=f"{key}-node", - binding_type=WorkflowAgentBindingType.INLINE_AGENT, - agent_id=f"{key}-inline-agent", - current_snapshot_id=f"{key}-snapshot", - node_job_config={}, - ) - for key in ("tenant", "app", "workflow", "version") - ] - sqlite_session.add_all([workflow, inline_binding, roster_binding, *non_target_bindings]) - sqlite_session.commit() - - result = workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) - sqlite_session.flush() - - assert result == ["inline-agent"] - assert sqlite_session.get(Workflow, workflow_id) is None - assert sqlite_session.get(WorkflowAgentNodeBinding, inline_binding.id) is None - assert sqlite_session.get(WorkflowAgentNodeBinding, roster_binding.id) is None - for binding in non_target_bindings: - assert sqlite_session.get(WorkflowAgentNodeBinding, binding.id) is binding - - def test_delete_workflow_locks_source_until_caller_commits( - self, workflow_service: WorkflowService, sqlite_session: Session, mocker: MockerFixture - ): - workflow = TestWorkflowAssociatedDataFactory.create_workflow(version="v1") - workflow_ref = WorkflowRef( - tenant_id=workflow.tenant_id, - owner_id=workflow.app_id, - workflow_id=workflow.id, - ) - session = sqlite_session - session.add(workflow) - session.commit() - lookup = mocker.spy(session, "scalar") - - result = workflow_service.delete_workflow(session=session, workflow_ref=workflow_ref) - - stmt = lookup.call_args_list[0].args[0] - sql = str(stmt.compile(dialect=postgresql.dialect())) - assert result == [] - assert "FOR UPDATE" in sql - assert session.in_transaction() - assert workflow in session.deleted - session.commit() - assert session.get(Workflow, workflow.id) is None - - def test_delete_workflow_with_ref_scopes_lookup_to_app( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """Test delete_workflow includes the trusted app owner in the lookup.""" - workflow_id = "workflow-123" - tenant_id = "tenant-456" - app_id = "app-789" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="other-app", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=app_id, version="v1" - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - with pytest.raises(ValueError, match="not found"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) - - assert sqlite_session.get(Workflow, workflow_id) is workflow - - def test_delete_workflow_draft_raises_error(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test delete_workflow raises error when trying to delete draft. - - Draft workflows cannot be deleted - they're the working copy. - Users can only delete published versions to clean up old snapshots. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, - tenant_id=tenant_id, - app_id=workflow_ref.owner_id, - version=Workflow.VERSION_DRAFT, - ) - sqlite_session.add(workflow) - sqlite_session.commit() - - with pytest.raises(DraftWorkflowDeletionError, match="Cannot delete draft workflow"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + def test_validate_workflow_credentials_should_raise_for_llm_node_missing_model( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """LLM nodes without provider AND name should raise ValueError.""" + # Arrange + nodes = [ + { + "id": "llm-node", + "data": {"type": "llm", "model": {"provider": "openai"}}, # name missing + } + ] + workflow = self._make_workflow(nodes) - def test_delete_workflow_in_use_by_app_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test delete_workflow raises error when workflow is in use by app. - - Cannot delete a workflow version that's currently published/active. - This would break the app for users. Must publish a different version first. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=workflow_ref.owner_id, version="v1" - ) - app = App( - id="active-app", - tenant_id=tenant_id, - name="Active App", - description="", - mode=AppMode.WORKFLOW, - workflow_id=workflow_id, - enable_site=True, - enable_api=True, - max_active_requests=0, - ) - sqlite_session.add_all([workflow, app]) - sqlite_session.commit() + # Act + Assert + with pytest.raises(ValueError, match="Missing provider or model configuration"): + service._validate_workflow_credentials(workflow, credentials=self.credentials) - with pytest.raises(WorkflowInUseError, match="currently in use by app"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + def test_validate_workflow_credentials_should_wrap_unexpected_exception_in_value_error( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """Non-ValueError exceptions from validation must be re-raised as ValueError.""" + # Arrange + nodes = [ + { + "id": "llm-node", + "data": { + "type": "llm", + "model": {"provider": "openai", "name": "gpt-4"}, + }, + } + ] + workflow = self._make_workflow(nodes) - def test_delete_workflow_published_as_tool_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test delete_workflow raises error when workflow is published as tool. - - Workflows can be published as reusable tools for other workflows. - Cannot delete a version that's being used as a tool, as this would - break other workflows that depend on it. - """ - workflow_id = "workflow-123" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id=workflow_id, tenant_id=tenant_id, app_id=workflow_ref.owner_id, version="v1" - ) - tool_provider = WorkflowToolProvider( - name="workflow-tool", - label="Workflow Tool", - icon="icon.svg", - app_id=workflow.app_id, - version=workflow.version, - user_id="user-123", - tenant_id=workflow.tenant_id, - description="Test provider", - parameter_configuration="[]", - ) - sqlite_session.add_all([workflow, tool_provider]) - sqlite_session.commit() + # Act + Assert + with patch.object(service, "_validate_llm_model_config", side_effect=RuntimeError("boom")): + with pytest.raises(ValueError, match="boom"): + service._validate_workflow_credentials(workflow, credentials=self.credentials) - with pytest.raises(WorkflowInUseError, match="published as a tool"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) + def test_validate_workflow_credentials_should_validate_agent_node_model( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + nodes = [ + { + "id": "agent-node", + "data": { + "type": "agent", + "agent_parameters": { + "model": {"value": {"provider": "openai", "model": "gpt-4"}}, + "tools": {"value": []}, + }, + }, + } + ] + workflow = self._make_workflow(nodes) - def test_delete_workflow_not_found_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session + # Act + with ( + patch.object(service, "_validate_llm_model_config") as mock_llm, + patch.object(service, "_validate_load_balancing_credentials"), ): - """Test delete_workflow raises error when workflow not found.""" - workflow_id = "nonexistent" - tenant_id = "tenant-456" - workflow_ref = WorkflowRef(tenant_id=tenant_id, owner_id="app-789", workflow_id=workflow_id) - - with pytest.raises(ValueError, match="not found"): - workflow_service.delete_workflow(session=sqlite_session, workflow_ref=workflow_ref) - - # ==================== Get Default Block Config Tests ==================== - # These tests verify retrieval of default node configurations - - def test_get_default_block_configs(self, workflow_service: WorkflowService): - """ - Test get_default_block_configs returns list of default configs. - - Returns default configurations for all available node types. - Used by the UI to populate the node palette and provide sensible defaults - when users add new nodes to their workflow. - """ - with patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping: - # Mock node class with default config - mock_node_class = MagicMock() - mock_node_class.get_default_config.return_value = {"type": "llm", "config": {}} - - mock_mapping.return_value = {BuiltinNodeTypes.LLM: {"latest": mock_node_class}} - - with patch("services.workflow_service.LATEST_VERSION", "latest"): - result = workflow_service.get_default_block_configs() - - assert len(result) > 0 - - def test_get_default_block_configs_http_request_injects_default_config(self, workflow_service: WorkflowService): - injected_config = HttpRequestNodeConfig( - max_connect_timeout=15, - max_read_timeout=25, - max_write_timeout=35, - max_binary_size=4096, - max_text_size=2048, - ssl_verify=True, - ssrf_default_max_retries=6, - ) - - with ( - patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, - patch("services.workflow_service.LATEST_VERSION", "latest"), - patch( - "services.workflow_service.build_http_request_config", - return_value=injected_config, - ) as mock_build_config, - ): - mock_http_node_class = MagicMock() - mock_http_node_class.get_default_config.return_value = {"type": "http-request", "config": {}} - mock_llm_node_class = MagicMock() - mock_llm_node_class.get_default_config.return_value = {"type": "llm", "config": {}} - mock_mapping.return_value = { - BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_http_node_class}, - BuiltinNodeTypes.LLM: {"latest": mock_llm_node_class}, - } - - result = workflow_service.get_default_block_configs() - - assert result == [ - {"type": "http-request", "config": {}}, - {"type": "llm", "config": {}}, - ] - mock_build_config.assert_called_once() - passed_http_filters = mock_http_node_class.get_default_config.call_args.kwargs["filters"] - assert passed_http_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is injected_config - mock_llm_node_class.get_default_config.assert_called_once_with(filters=None) - - def test_get_default_block_config_for_node_type(self, workflow_service: WorkflowService): - """ - Test get_default_block_config returns config for specific node type. - - Returns the default configuration for a specific node type (e.g., LLM, HTTP). - This includes default values for all required and optional parameters. - """ - with ( - patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, - patch("services.workflow_service.LATEST_VERSION", "latest"), - ): - # Mock node class with default config - mock_node_class = MagicMock() - mock_config = {"type": "llm", "config": {"provider": "openai"}} - mock_node_class.get_default_config.return_value = mock_config - - # Create a mock mapping that includes BuiltinNodeTypes.LLM - mock_mapping.return_value = {BuiltinNodeTypes.LLM: {"latest": mock_node_class}} - - result = workflow_service.get_default_block_config(BuiltinNodeTypes.LLM) - - assert result == mock_config - mock_node_class.get_default_config.assert_called_once() - - def test_get_default_block_config_invalid_node_type(self, workflow_service: WorkflowService): - """Test get_default_block_config returns empty dict for invalid node type.""" - with patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping: - mock_mapping.return_value = {} - - # Use a valid NodeType but one that's not in the mapping - result = workflow_service.get_default_block_config(BuiltinNodeTypes.LLM) - - assert result == {} - - def test_get_default_block_config_http_request_injects_default_config(self, workflow_service: WorkflowService): - injected_config = HttpRequestNodeConfig( - max_connect_timeout=11, - max_read_timeout=22, - max_write_timeout=33, - max_binary_size=4096, - max_text_size=2048, - ssl_verify=False, - ssrf_default_max_retries=7, - ) - - with ( - patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, - patch("services.workflow_service.LATEST_VERSION", "latest"), - patch( - "services.workflow_service.build_http_request_config", - return_value=injected_config, - ) as mock_build_config, - ): - mock_node_class = MagicMock() - expected = {"type": "http-request", "config": {}} - mock_node_class.get_default_config.return_value = expected - mock_mapping.return_value = {BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_node_class}} - - result = workflow_service.get_default_block_config(BuiltinNodeTypes.HTTP_REQUEST) - - assert result == expected - mock_build_config.assert_called_once() - passed_filters = mock_node_class.get_default_config.call_args.kwargs["filters"] - assert passed_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is injected_config - - def test_get_default_block_config_http_request_uses_passed_config(self, workflow_service: WorkflowService): - provided_config = HttpRequestNodeConfig( - max_connect_timeout=13, - max_read_timeout=23, - max_write_timeout=34, - max_binary_size=8192, - max_text_size=4096, - ssl_verify=True, - ssrf_default_max_retries=2, - ) - - with ( - patch("services.workflow_service.get_node_type_classes_mapping") as mock_mapping, - patch("services.workflow_service.LATEST_VERSION", "latest"), - patch("services.workflow_service.build_http_request_config") as mock_build_config, - ): - mock_node_class = MagicMock() - expected = {"type": "http-request", "config": {}} - mock_node_class.get_default_config.return_value = expected - mock_mapping.return_value = {BuiltinNodeTypes.HTTP_REQUEST: {"latest": mock_node_class}} + service._validate_workflow_credentials(workflow, credentials=self.credentials) - result = workflow_service.get_default_block_config( - BuiltinNodeTypes.HTTP_REQUEST, - filters={HTTP_REQUEST_CONFIG_FILTER_KEY: provided_config}, - ) - - assert result == expected - mock_build_config.assert_not_called() - passed_filters = mock_node_class.get_default_config.call_args.kwargs["filters"] - assert passed_filters[HTTP_REQUEST_CONFIG_FILTER_KEY] is provided_config - - def test_get_default_block_config_http_request_malformed_config_raises_type_error( - self, workflow_service: WorkflowService - ): - with ( - patch( - "services.workflow_service.get_node_type_classes_mapping", - return_value={BuiltinNodeTypes.HTTP_REQUEST: {"latest": HttpRequestNode}}, - ), - patch("services.workflow_service.LATEST_VERSION", "latest"), - ): - with pytest.raises(TypeError, match="http_request_config must be an HttpRequestNodeConfig instance"): - workflow_service.get_default_block_config( - BuiltinNodeTypes.HTTP_REQUEST, - filters={HTTP_REQUEST_CONFIG_FILTER_KEY: "invalid"}, - ) + # Assert + mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") - # ==================== Workflow Conversion Tests ==================== - # These tests verify converting basic apps to workflow apps - - def test_convert_to_workflow_from_chat_app(self, workflow_service: WorkflowService, sqlite_session: Session): - """ - Test convert_to_workflow converts chat app to workflow. - - Allows users to migrate from simple chat apps to advanced workflow apps. - The conversion creates equivalent workflow nodes from the chat configuration, - giving users more control and customization options. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.CHAT) - account = TestWorkflowAssociatedDataFactory.create_account() - args = { - "name": "Converted Workflow", - "icon_type": "emoji", - "icon": "🤖", - "icon_background": "#FFEAD5", + def test_validate_workflow_credentials_should_validate_agent_tools( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """Each agent tool with a provider should be checked for credential compliance.""" + # Arrange + nodes = [ + { + "id": "agent-node", + "data": { + "type": "agent", + "agent_parameters": { + "model": {"value": {}}, # no model config + "tools": { + "value": [ + {"provider_name": "provider-a", "credential_id": "cred-a"}, + {"provider_name": "provider-b"}, # uses default + ] + }, + }, + }, } + ] + workflow = self._make_workflow(nodes) - with patch("services.workflow_service.WorkflowConverter") as mock_converter_type: - mock_converter = mock_converter_type.return_value - mock_new_app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - mock_converter.convert_to_workflow.return_value = mock_new_app - - result = workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) - - assert result == mock_new_app - mock_converter.convert_to_workflow.assert_called_once() - - def test_convert_to_workflow_from_completion_app( - self, workflow_service: WorkflowService, sqlite_session: Session + # Act + with ( + patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check, + patch.object(service, "_check_default_tool_credential") as mock_default, ): - """ - Test convert_to_workflow converts completion app to workflow. + service._validate_workflow_credentials(workflow, credentials=self.credentials) - Similar to chat conversion, but for completion-style apps. - Completion apps are simpler (single prompt-response), so the - conversion creates a basic workflow with fewer nodes. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.COMPLETION) - account = TestWorkflowAssociatedDataFactory.create_account() - args = {"name": "Converted Workflow"} + # Assert + mock_check.assert_called_once() # provider-a has credential_id + mock_default.assert_called_once_with("tenant-1", "provider-b", credentials=self.credentials) - with patch("services.workflow_service.WorkflowConverter") as mock_converter_type: - mock_converter = mock_converter_type.return_value - mock_new_app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - mock_converter.convert_to_workflow.return_value = mock_new_app + # --- _validate_llm_model_config --- - result = workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) + def test_validate_llm_model_config_should_raise_value_error_on_failure(self, service: WorkflowService) -> None: + """If ModelManager raises any exception it must be wrapped into ValueError.""" + # Arrange + assembly = MagicMock() + assembly.model_manager.get_model_instance.side_effect = RuntimeError("no key") - assert result == mock_new_app + with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): + # Act + Assert + with pytest.raises(ValueError, match="Failed to validate LLM model configuration"): + service._validate_llm_model_config("tenant-1", "openai", "gpt-4") - def test_convert_to_workflow_invalid_mode_raises_error( - self, workflow_service: WorkflowService, sqlite_session: Session - ): - """ - Test convert_to_workflow raises error for invalid app mode. - - Only chat and completion apps can be converted to workflows. - Apps that are already workflows or have other modes cannot be converted. - """ - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - account = TestWorkflowAssociatedDataFactory.create_account() - args = {} - - with pytest.raises(ValueError, match="not supported convert to workflow"): - workflow_service.convert_to_workflow(app, account, args, session=sqlite_session) - - # =========================================================================== - # TestWorkflowServiceCredentialValidation - # Tests for _validate_workflow_credentials and related private helpers - # =========================================================================== - - @pytest.mark.usefixtures("sqlite_session") - class TestWorkflowServiceCredentialValidation: - """ - Tests for the private credential-validation helpers on WorkflowService. + def test_validate_llm_model_config_success(self, service: WorkflowService) -> None: + """Test success path with ProviderManager and Model entities.""" + mock_model = MagicMock() + mock_model.model = "gpt-4" + mock_model.provider.provider = "openai" - These helpers gate `publish_workflow` when `PluginManager` is enabled. - Each test focuses on a distinct branch inside `_validate_workflow_credentials`, - `_validate_llm_model_config`, `_check_default_tool_credential`, and the - load-balancing path. - """ + mock_configs = MagicMock() + mock_configs.get_models.return_value = [mock_model] + assembly = MagicMock() + assembly.provider_manager.get_configurations.return_value = mock_configs - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) + with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): + # Act + service._validate_llm_model_config("tenant-1", "openai", "gpt-4") - @staticmethod - def _make_workflow(nodes: list[dict]) -> Workflow: - return TestWorkflowAssociatedDataFactory.create_workflow( + # Assert + mock_model.raise_for_status.assert_called_once() + assembly.model_manager.get_model_instance.assert_called_once_with( tenant_id="tenant-1", - app_id="app-1", - graph={"nodes": nodes, "edges": []}, + provider="openai", + model_type=ModelType.LLM, + model="gpt-4", ) - # --- _validate_workflow_credentials: tool node (with credential_id) --- - - def test_validate_workflow_credentials_should_check_tool_credential_when_credential_id_present( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - nodes = [ - { - "id": "tool-node", - "data": { - "type": "tool", - "provider_id": "my-provider", - "credential_id": "cred-123", - }, - } - ] - workflow = self._make_workflow(nodes) + def test_validate_llm_model_config_model_not_found(self, service: WorkflowService) -> None: + """Test ValueError when model is not found in provider configurations.""" + mock_configs = MagicMock() + mock_configs.get_models.return_value = [] # No models + assembly = MagicMock() + assembly.provider_manager.get_configurations.return_value = mock_configs + with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): # Act + Assert - with patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check: - # Should not raise; mock allows the call - service._validate_workflow_credentials(workflow, session=sqlite_session) - mock_check.assert_called_once() - - def test_validate_workflow_credentials_should_check_default_credential_when_no_credential_id( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - nodes = [ - { - "id": "tool-node", - "data": { - "type": "tool", - "provider_id": "my-provider", - # No credential_id — should fall back to default - }, - } - ] - workflow = self._make_workflow(nodes) + with pytest.raises(ValueError, match="Model gpt-4 not found for provider openai"): + service._validate_llm_model_config("tenant-1", "openai", "gpt-4") - # Act - with patch.object(service, "_check_default_tool_credential") as mock_default: - service._validate_workflow_credentials(workflow, session=sqlite_session) + # --- _check_default_tool_credential --- - # Assert - mock_default.assert_called_once_with("tenant-1", "my-provider", session=sqlite_session) - - def test_validate_workflow_credentials_should_skip_tool_node_without_provider( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Tool nodes without a provider_id should be silently skipped.""" - # Arrange - nodes = [{"id": "tool-node", "data": {"type": "tool"}}] - workflow = self._make_workflow(nodes) - - # Act + Assert (no error raised) - with patch.object(service, "_check_default_tool_credential") as mock_default: - service._validate_workflow_credentials(workflow, session=sqlite_session) - mock_default.assert_not_called() - - def test_validate_workflow_credentials_should_validate_llm_node_with_model_config( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - nodes = [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": {"provider": "openai", "name": "gpt-4"}, - }, - } - ] - workflow = self._make_workflow(nodes) + def test_check_default_tool_credential_should_silently_pass_when_no_provider_found( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """Missing BuiltinToolProvider → plugin requires no credentials → no error.""" + service._check_default_tool_credential("tenant-1", "some-provider", credentials=self.credentials) - # Act - with ( - patch.object(service, "_validate_llm_model_config") as mock_llm, - patch.object(service, "_validate_load_balancing_credentials"), - ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + def test_check_default_tool_credential_should_raise_when_compliance_fails( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + provider = BuiltinToolProvider( + name="API key", + tenant_id="tenant-1", + user_id="user-1", + provider="some-provider", + ) + sqlite_session.add(provider) + sqlite_session.commit() - # Assert - mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") - - def test_validate_workflow_credentials_should_use_llm_environment_variable_model( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - workflow = self._make_workflow( - [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": { - "provider": "old-provider", - "name": "old-model", - "mode": "chat", - "completion_params": {"temperature": 0.2}, - }, - "model_selector": ["env", "shared_model"], - }, - } - ] - ) - workflow.environment_variables = [ - LLMEnvironmentVariable( - name="shared_model", - value={ - "provider": "new-provider", - "name": "new-model", - "mode": "chat", - "completion_params": {"temperature": 0.8}, - }, - ) - ] + with patch("core.helper.credential_utils.check_credential_policy_compliance", side_effect=Exception("denied")): + with pytest.raises(ValueError, match="Failed to validate default credential"): + service._check_default_tool_credential("tenant-1", "some-provider", credentials=self.credentials) - with ( - patch.object(service, "_validate_llm_model_config") as validate_model, - patch.object(service, "_validate_load_balancing_credentials") as validate_load_balancing, - ): - service._validate_workflow_credentials(workflow, session=sqlite_session) - - validate_model.assert_called_once_with("tenant-1", "new-provider", "new-model") - validated_node_data = validate_load_balancing.call_args.args[1] - assert validated_node_data["model"] == { - "provider": "new-provider", - "name": "new-model", - "mode": "chat", - "completion_params": {"temperature": 0.8}, - } + # --- _is_load_balancing_enabled --- - def test_validate_workflow_credentials_should_reject_llm_environment_variable_mode_mismatch( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - workflow = self._make_workflow( - [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": {"provider": "provider", "name": "model", "mode": "chat"}, - "model_selector": ["env", "shared_model"], - }, - } - ] - ) - workflow.environment_variables = [ - LLMEnvironmentVariable( - name="shared_model", - value={"provider": "provider", "name": "model", "mode": "completion"}, - ) - ] + def test_is_load_balancing_enabled_should_return_false_when_provider_not_found( + self, service: WorkflowService + ) -> None: + with patch("core.provider_manager.ProviderManager.get_configurations") as mock_get_configs: + mock_configs = MagicMock() + mock_configs.get.return_value = None # provider not found + mock_get_configs.return_value = mock_configs - with pytest.raises(ValueError, match="uses mode 'completion'.*uses mode 'chat'"): - service._validate_workflow_credentials(workflow, session=sqlite_session) + # Act + result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") - def test_validate_workflow_credentials_should_raise_for_llm_node_missing_model( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """LLM nodes without provider AND name should raise ValueError.""" - # Arrange - nodes = [ - { - "id": "llm-node", - "data": {"type": "llm", "model": {"provider": "openai"}}, # name missing - } - ] - workflow = self._make_workflow(nodes) + # Assert + assert result is False - # Act + Assert - with pytest.raises(ValueError, match="Missing provider or model configuration"): - service._validate_workflow_credentials(workflow, session=sqlite_session) - - def test_validate_workflow_credentials_should_wrap_unexpected_exception_in_value_error( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Non-ValueError exceptions from validation must be re-raised as ValueError.""" - # Arrange - nodes = [ - { - "id": "llm-node", - "data": { - "type": "llm", - "model": {"provider": "openai", "name": "gpt-4"}, - }, - } - ] - workflow = self._make_workflow(nodes) + def test_is_load_balancing_enabled_should_return_true_when_setting_enabled(self, service: WorkflowService) -> None: + # Arrange + with patch("core.provider_manager.ProviderManager.get_configurations") as mock_get_configs: + mock_provider_config = MagicMock() + mock_provider_model_setting = MagicMock() + mock_provider_model_setting.load_balancing_enabled = True + mock_provider_config.get_provider_model_setting.return_value = mock_provider_model_setting - # Act + Assert - with patch.object(service, "_validate_llm_model_config", side_effect=RuntimeError("boom")): - with pytest.raises(ValueError, match="boom"): - service._validate_workflow_credentials(workflow, session=sqlite_session) - - def test_validate_workflow_credentials_should_validate_agent_node_model( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - nodes = [ - { - "id": "agent-node", - "data": { - "type": "agent", - "agent_parameters": { - "model": {"value": {"provider": "openai", "model": "gpt-4"}}, - "tools": {"value": []}, - }, - }, - } - ] - workflow = self._make_workflow(nodes) + mock_configs = MagicMock() + mock_configs.get.return_value = mock_provider_config + mock_get_configs.return_value = mock_configs # Act - with ( - patch.object(service, "_validate_llm_model_config") as mock_llm, - patch.object(service, "_validate_load_balancing_credentials"), - ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") - # Assert - mock_llm.assert_called_once_with("tenant-1", "openai", "gpt-4") - - def test_validate_workflow_credentials_should_validate_agent_tools( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Each agent tool with a provider should be checked for credential compliance.""" - # Arrange - nodes = [ - { - "id": "agent-node", - "data": { - "type": "agent", - "agent_parameters": { - "model": {"value": {}}, # no model config - "tools": { - "value": [ - {"provider_name": "provider-a", "credential_id": "cred-a"}, - {"provider_name": "provider-b"}, # uses default - ] - }, - }, - }, - } - ] - workflow = self._make_workflow(nodes) + # Assert + assert result is True + def test_is_load_balancing_enabled_should_return_false_on_exception(self, service: WorkflowService) -> None: + """Any exception should be swallowed and return False.""" + # Arrange + with patch("core.provider_manager.ProviderManager.get_configurations", side_effect=RuntimeError("db down")): # Act - with ( - patch("core.helper.credential_utils.check_credential_policy_compliance") as mock_check, - patch.object(service, "_check_default_tool_credential") as mock_default, - ): - service._validate_workflow_credentials(workflow, session=sqlite_session) + result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") - # Assert - mock_check.assert_called_once() # provider-a has credential_id - mock_default.assert_called_once_with("tenant-1", "provider-b", session=sqlite_session) + # Assert + assert result is False - # --- _validate_llm_model_config --- + # --- _validate_load_balancing_credentials --- - def test_validate_llm_model_config_should_raise_value_error_on_failure(self, service: WorkflowService) -> None: - """If ModelManager raises any exception it must be wrapped into ValueError.""" - # Arrange - assembly = MagicMock() - assembly.model_manager.get_model_instance.side_effect = RuntimeError("no key") + def test_validate_load_balancing_credentials_should_skip_when_no_model_config( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + """Missing provider or model in node_data should be a no-op.""" + # Arrange + workflow = self._make_workflow([]) + node_data: dict[str, Any] = {} # no model key - with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): - # Act + Assert - with pytest.raises(ValueError, match="Failed to validate LLM model configuration"): - service._validate_llm_model_config("tenant-1", "openai", "gpt-4") + # Act + Assert (no error expected) + service._validate_load_balancing_credentials(workflow, node_data, "node-1", credentials=self.credentials) - def test_validate_llm_model_config_success(self, service: WorkflowService) -> None: - """Test success path with ProviderManager and Model entities.""" - mock_model = MagicMock() - mock_model.model = "gpt-4" - mock_model.provider.provider = "openai" + def test_validate_load_balancing_credentials_should_skip_when_lb_not_enabled( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + workflow = self._make_workflow([]) + node_data = {"model": {"provider": "openai", "name": "gpt-4"}} - mock_configs = MagicMock() - mock_configs.get_models.return_value = [mock_model] - assembly = MagicMock() - assembly.provider_manager.get_configurations.return_value = mock_configs + # Act + Assert (no error expected) + with patch.object(service, "_is_load_balancing_enabled", return_value=False): + service._validate_load_balancing_credentials(workflow, node_data, "node-1", credentials=self.credentials) - with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): - # Act - service._validate_llm_model_config("tenant-1", "openai", "gpt-4") + def test_validate_load_balancing_credentials_should_raise_when_compliance_fails( + self, service: WorkflowService, sqlite_session: Session + ) -> None: + # Arrange + workflow = self._make_workflow([]) + node_data = {"model": {"provider": "openai", "name": "gpt-4"}} + lb_configs = ["cred-lb-1"] - # Assert - mock_model.raise_for_status.assert_called_once() - assembly.model_manager.get_model_instance.assert_called_once_with( - tenant_id="tenant-1", - provider="openai", - model_type=ModelType.LLM, - model="gpt-4", + # Act + Assert + with ( + patch.object(service, "_is_load_balancing_enabled", return_value=True), + patch.object(self.credentials, "list_model_load_balancing_credential_ids", return_value=lb_configs), + patch( + "core.helper.credential_utils.check_credential_policy_compliance", + side_effect=Exception("policy violation"), + ), + ): + with pytest.raises(ValueError, match="Invalid load balancing credentials"): + service._validate_load_balancing_credentials( + workflow, node_data, "node-1", credentials=self.credentials ) - def test_validate_llm_model_config_model_not_found(self, service: WorkflowService) -> None: - """Test ValueError when model is not found in provider configurations.""" - mock_configs = MagicMock() - mock_configs.get_models.return_value = [] # No models - assembly = MagicMock() - assembly.provider_manager.get_configurations.return_value = mock_configs - - with patch("services.workflow_service.create_plugin_model_assembly", return_value=assembly): - # Act + Assert - with pytest.raises(ValueError, match="Model gpt-4 not found for provider openai"): - service._validate_llm_model_config("tenant-1", "openai", "gpt-4") - - # --- _check_default_tool_credential --- - - def test_check_default_tool_credential_should_silently_pass_when_no_provider_found( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Missing BuiltinToolProvider → plugin requires no credentials → no error.""" - service._check_default_tool_credential("tenant-1", "some-provider", session=sqlite_session) - - def test_check_default_tool_credential_should_raise_when_compliance_fails( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - provider = BuiltinToolProvider( - name="API key", - tenant_id="tenant-1", - user_id="user-1", - provider="some-provider", - ) - sqlite_session.add(provider) - sqlite_session.commit() - - with patch( - "core.helper.credential_utils.check_credential_policy_compliance", side_effect=Exception("denied") - ): - with pytest.raises(ValueError, match="Failed to validate default credential"): - service._check_default_tool_credential("tenant-1", "some-provider", session=sqlite_session) - # --- _is_load_balancing_enabled --- +# =========================================================================== +# TestWorkflowServiceExecutionHelpers +# Tests for _apply_error_strategy, _populate_execution_result, _execute_node_safely +# =========================================================================== - def test_is_load_balancing_enabled_should_return_false_when_provider_not_found( - self, service: WorkflowService - ) -> None: - with patch("core.provider_manager.ProviderManager.get_configurations") as mock_get_configs: - mock_configs = MagicMock() - mock_configs.get.return_value = None # provider not found - mock_get_configs.return_value = mock_configs - # Act - result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") - - # Assert - assert result is False - - def test_is_load_balancing_enabled_should_return_true_when_setting_enabled( - self, service: WorkflowService - ) -> None: - # Arrange - with patch("core.provider_manager.ProviderManager.get_configurations") as mock_get_configs: - mock_provider_config = MagicMock() - mock_provider_model_setting = MagicMock() - mock_provider_model_setting.load_balancing_enabled = True - mock_provider_config.get_provider_model_setting.return_value = mock_provider_model_setting +class TestWorkflowServiceExecutionHelpers: + """ + Tests for the private execution-result handling methods: + _apply_error_strategy, _populate_execution_result, _execute_node_safely. + """ - mock_configs = MagicMock() - mock_configs.get.return_value = mock_provider_config - mock_get_configs.return_value = mock_configs + @pytest.fixture + def service(self, sqlite_engine: Engine) -> WorkflowService: + return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) + + # --- _apply_error_strategy --- + + def test_apply_error_strategy_should_return_exception_status_noderunresult(self, service: WorkflowService) -> None: + # Arrange + node = MagicMock() + node.error_strategy = ErrorStrategy.FAIL_BRANCH + node.default_value_dict = {} + original = NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error="something went wrong", + error_type="SomeError", + inputs={"x": 1}, + outputs={}, + ) - # Act - result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") + # Act + result = service._apply_error_strategy(node, original) - # Assert - assert result is True + # Assert + assert result.status == WorkflowNodeExecutionStatus.EXCEPTION + assert result.error == "something went wrong" + assert result.metadata[WorkflowNodeExecutionMetadataKey.ERROR_STRATEGY] == ErrorStrategy.FAIL_BRANCH - def test_is_load_balancing_enabled_should_return_false_on_exception(self, service: WorkflowService) -> None: - """Any exception should be swallowed and return False.""" - # Arrange - with patch("core.provider_manager.ProviderManager.get_configurations", side_effect=RuntimeError("db down")): - # Act - result = service._is_load_balancing_enabled("tenant-1", "openai", "gpt-4") + def test_apply_error_strategy_should_include_default_values_for_default_value_strategy( + self, service: WorkflowService + ) -> None: + # Arrange + node = MagicMock() + node.error_strategy = ErrorStrategy.DEFAULT_VALUE + node.default_value_dict = {"output_key": "fallback"} + original = NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error="err", + ) - # Assert - assert result is False - - # --- _get_load_balancing_configs --- - - def test_get_load_balancing_configs_should_return_empty_list_on_exception( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Any exception during LB config retrieval should return an empty list.""" - # Arrange - with patch( - "services.model_load_balancing_service.ModelLoadBalancingService.get_load_balancing_configs", - side_effect=RuntimeError("fail"), - ): - # Act - result = service._get_load_balancing_configs("tenant-1", "openai", "gpt-4", session=sqlite_session) + # Act + result = service._apply_error_strategy(node, original) - # Assert - assert result == [] - - def test_get_load_balancing_configs_should_merge_predefined_and_custom( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - predefined = [{"credential_id": "cred-a"}, {"credential_id": None}] - custom = [{"credential_id": "cred-b"}] - with patch( - "services.model_load_balancing_service.ModelLoadBalancingService.get_load_balancing_configs", - side_effect=[ - (None, predefined), # first call: predefined-model - (None, custom), # second call: custom-model - ], - ): - # Act - result = service._get_load_balancing_configs("tenant-1", "openai", "gpt-4", session=sqlite_session) - - # Assert — only entries with a credential_id should be returned - assert len(result) == 2 - assert all(c["credential_id"] for c in result) - - # --- _validate_load_balancing_credentials --- - - def test_validate_load_balancing_credentials_should_skip_when_no_model_config( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - """Missing provider or model in node_data should be a no-op.""" - # Arrange - node_data: dict[str, Any] = {} # no model key - - # Act + Assert (no error expected) - service._validate_load_balancing_credentials("tenant-1", node_data, "node-1", session=sqlite_session) - - def test_validate_load_balancing_credentials_should_skip_when_lb_not_enabled( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - node_data = {"model": {"provider": "openai", "name": "gpt-4"}} - - # Act + Assert (no error expected) - with patch.object(service, "_is_load_balancing_enabled", return_value=False): - service._validate_load_balancing_credentials("tenant-1", node_data, "node-1", session=sqlite_session) - - def test_validate_load_balancing_credentials_should_raise_when_compliance_fails( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - # Arrange - node_data = {"model": {"provider": "openai", "name": "gpt-4"}} - lb_configs = [{"credential_id": "cred-lb-1"}] + # Assert + assert result.outputs.get("output_key") == "fallback" + assert result.status == WorkflowNodeExecutionStatus.EXCEPTION - # Act + Assert - with ( - patch.object(service, "_is_load_balancing_enabled", return_value=True), - patch.object(service, "_get_load_balancing_configs", return_value=lb_configs), - patch( - "core.helper.credential_utils.check_credential_policy_compliance", - side_effect=Exception("policy violation"), - ), - ): - with pytest.raises(ValueError, match="Invalid load balancing credentials"): - service._validate_load_balancing_credentials( - "tenant-1", node_data, "node-1", session=sqlite_session - ) + # --- _populate_execution_result --- - # =========================================================================== - # TestWorkflowServiceExecutionHelpers - # Tests for _apply_error_strategy, _populate_execution_result, _execute_node_safely - # =========================================================================== + def test_populate_execution_result_should_set_succeeded_fields_when_run_succeeded( + self, service: WorkflowService + ) -> None: + # Arrange + node_execution = MagicMock(error=None) + node_run_result = NodeRunResult( + status=WorkflowNodeExecutionStatus.SUCCEEDED, + inputs={"q": "hello"}, + process_data={"steps": 3}, + outputs={"answer": "hi"}, + metadata={WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: 10}, + ) - class TestWorkflowServiceExecutionHelpers: - """ - Tests for the private execution-result handling methods: - _apply_error_strategy, _populate_execution_result, _execute_node_safely. - """ + # Act + with patch("services.workflow_service.WorkflowEntry.handle_special_values", side_effect=lambda x: x): + service._populate_execution_result(node_execution, node_run_result, True, None) - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - # --- _apply_error_strategy --- - - def test_apply_error_strategy_should_return_exception_status_noderunresult( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - node.error_strategy = ErrorStrategy.FAIL_BRANCH - node.default_value_dict = {} - original = NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - error="something went wrong", - error_type="SomeError", - inputs={"x": 1}, - outputs={}, - ) + # Assert + assert node_execution.status == WorkflowNodeExecutionStatus.SUCCEEDED + assert node_execution.outputs == {"answer": "hi"} + assert node_execution.error is None # SUCCEEDED status doesn't set error - # Act - result = service._apply_error_strategy(node, original) + def test_populate_execution_result_should_set_failed_status_and_error_when_not_succeeded( + self, service: WorkflowService + ) -> None: + # Arrange + node_execution = MagicMock(error=None) - # Assert - assert result.status == WorkflowNodeExecutionStatus.EXCEPTION - assert result.error == "something went wrong" - assert result.metadata[WorkflowNodeExecutionMetadataKey.ERROR_STRATEGY] == ErrorStrategy.FAIL_BRANCH - - def test_apply_error_strategy_should_include_default_values_for_default_value_strategy( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - node.error_strategy = ErrorStrategy.DEFAULT_VALUE - node.default_value_dict = {"output_key": "fallback"} - original = NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - error="err", - ) + # Act + service._populate_execution_result(node_execution, None, False, "catastrophic failure") - # Act - result = service._apply_error_strategy(node, original) + # Assert + assert node_execution.status == WorkflowNodeExecutionStatus.FAILED + assert node_execution.error == "catastrophic failure" - # Assert - assert result.outputs.get("output_key") == "fallback" - assert result.status == WorkflowNodeExecutionStatus.EXCEPTION - - # --- _populate_execution_result --- - - def test_populate_execution_result_should_set_succeeded_fields_when_run_succeeded( - self, service: WorkflowService - ) -> None: - # Arrange - node_execution = MagicMock(error=None) - node_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, - inputs={"q": "hello"}, - process_data={"steps": 3}, - outputs={"answer": "hi"}, - metadata={WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: 10}, - ) + def test_populate_execution_result_should_set_error_field_for_exception_status( + self, service: WorkflowService + ) -> None: + """A succeeded=True result with EXCEPTION status should still populate the error field.""" + # Arrange + node_execution = MagicMock() + node_run_result = NodeRunResult( + status=WorkflowNodeExecutionStatus.EXCEPTION, + error="constraint violated", + ) - # Act - with patch("services.workflow_service.WorkflowEntry.handle_special_values", side_effect=lambda x: x): - service._populate_execution_result(node_execution, node_run_result, True, None) + # Act + with patch("services.workflow_service.WorkflowEntry.handle_special_values", side_effect=lambda x: x): + service._populate_execution_result(node_execution, node_run_result, True, None) - # Assert - assert node_execution.status == WorkflowNodeExecutionStatus.SUCCEEDED - assert node_execution.outputs == {"answer": "hi"} - assert node_execution.error is None # SUCCEEDED status doesn't set error + # Assert + assert node_execution.status == WorkflowNodeExecutionStatus.EXCEPTION + assert node_execution.error == "constraint violated" - def test_populate_execution_result_should_set_failed_status_and_error_when_not_succeeded( - self, service: WorkflowService - ) -> None: - # Arrange - node_execution = MagicMock(error=None) + # --- _execute_node_safely --- - # Act - service._populate_execution_result(node_execution, None, False, "catastrophic failure") + def test_execute_node_safely_should_return_succeeded_result_on_happy_path(self, service: WorkflowService) -> None: + # Arrange + node = MagicMock() + node.error_strategy = None + node_run_result = NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED) + succeeded_event = NodeRunSucceededEvent( + id=str(uuid.uuid4()), + node_id="node-1", + node_type=BuiltinNodeTypes.LLM, + node_run_result=node_run_result, + start_at=naive_utc_now(), + ) - # Assert - assert node_execution.status == WorkflowNodeExecutionStatus.FAILED - assert node_execution.error == "catastrophic failure" - - def test_populate_execution_result_should_set_error_field_for_exception_status( - self, service: WorkflowService - ) -> None: - """A succeeded=True result with EXCEPTION status should still populate the error field.""" - # Arrange - node_execution = MagicMock() - node_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.EXCEPTION, - error="constraint violated", - ) + def invoke_fn(): + def _gen(): + yield succeeded_event - # Act - with patch("services.workflow_service.WorkflowEntry.handle_special_values", side_effect=lambda x: x): - service._populate_execution_result(node_execution, node_run_result, True, None) + return node, _gen() - # Assert - assert node_execution.status == WorkflowNodeExecutionStatus.EXCEPTION - assert node_execution.error == "constraint violated" - - # --- _execute_node_safely --- - - def test_execute_node_safely_should_return_succeeded_result_on_happy_path( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - node.error_strategy = None - node_run_result = NodeRunResult(status=WorkflowNodeExecutionStatus.SUCCEEDED) - succeeded_event = NodeRunSucceededEvent( - id=str(uuid.uuid4()), - node_id="node-1", - node_type=BuiltinNodeTypes.LLM, - node_run_result=node_run_result, - start_at=naive_utc_now(), - ) + # Act + out_node, out_result, run_succeeded, error = service._execute_node_safely(invoke_fn) - def invoke_fn(): - def _gen(): - yield succeeded_event + # Assert + assert out_node is node + assert run_succeeded is True + assert error is None - return node, _gen() + def test_execute_node_safely_should_return_failed_result_on_failed_event(self, service: WorkflowService) -> None: + # Arrange + node = MagicMock() + node.error_strategy = None + node_run_result = NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error="node exploded", + ) + failed_event = NodeRunFailedEvent( + id=str(uuid.uuid4()), + node_id="node-1", + node_type=BuiltinNodeTypes.LLM, + node_run_result=node_run_result, + start_at=naive_utc_now(), + error="node exploded", + ) - # Act - out_node, out_result, run_succeeded, error = service._execute_node_safely(invoke_fn) + def invoke_fn(): + def _gen(): + yield failed_event - # Assert - assert out_node is node - assert run_succeeded is True - assert error is None - - def test_execute_node_safely_should_return_failed_result_on_failed_event( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - node.error_strategy = None - node_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - error="node exploded", - ) - failed_event = NodeRunFailedEvent( - id=str(uuid.uuid4()), - node_id="node-1", - node_type=BuiltinNodeTypes.LLM, - node_run_result=node_run_result, - start_at=naive_utc_now(), - error="node exploded", - ) + return node, _gen() - def invoke_fn(): - def _gen(): - yield failed_event + # Act + _, _, run_succeeded, error = service._execute_node_safely(invoke_fn) - return node, _gen() + # Assert + assert run_succeeded is False + assert error == "node exploded" - # Act - _, _, run_succeeded, error = service._execute_node_safely(invoke_fn) + def test_execute_node_safely_should_handle_workflow_node_run_failed_error(self, service: WorkflowService) -> None: + # Arrange + node = MagicMock() + exc = WorkflowNodeRunFailedError(node, "runtime failure") - # Assert - assert run_succeeded is False - assert error == "node exploded" + def invoke_fn(): + raise exc - def test_execute_node_safely_should_handle_workflow_node_run_failed_error( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - exc = WorkflowNodeRunFailedError(node, "runtime failure") + # Act + out_node, out_result, run_succeeded, error = service._execute_node_safely(invoke_fn) - def invoke_fn(): - raise exc + # Assert + assert out_node is node + assert out_result is None + assert run_succeeded is False + assert error == "runtime failure" - # Act - out_node, out_result, run_succeeded, error = service._execute_node_safely(invoke_fn) + def test_execute_node_safely_should_raise_when_no_result_event(self, service: WorkflowService) -> None: + """If the generator produces no NodeRunSucceededEvent/NodeRunFailedEvent, ValueError is expected.""" + # Arrange + node = MagicMock() + node.error_strategy = None - # Assert - assert out_node is node - assert out_result is None - assert run_succeeded is False - assert error == "runtime failure" + def invoke_fn(): + def _gen(): + yield from [] - def test_execute_node_safely_should_raise_when_no_result_event(self, service: WorkflowService) -> None: - """If the generator produces no NodeRunSucceededEvent/NodeRunFailedEvent, ValueError is expected.""" - # Arrange - node = MagicMock() - node.error_strategy = None + return node, _gen() - def invoke_fn(): - def _gen(): - yield from [] + # Act + Assert + with pytest.raises(ValueError, match="no result returned"): + service._execute_node_safely(invoke_fn) - return node, _gen() + # --- _apply_error_strategy with FAIL_BRANCH strategy --- - # Act + Assert - with pytest.raises(ValueError, match="no result returned"): - service._execute_node_safely(invoke_fn) - - # --- _apply_error_strategy with FAIL_BRANCH strategy --- - - def test_execute_node_safely_should_apply_error_strategy_on_failed_status( - self, service: WorkflowService - ) -> None: - # Arrange - node = MagicMock() - node.error_strategy = ErrorStrategy.FAIL_BRANCH - node.default_value_dict = {} - - original_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - error="oops", - error_type="ValueError", - inputs={}, - ) - failed_event = NodeRunFailedEvent( - id=str(uuid.uuid4()), - node_id="node-1", - node_type=BuiltinNodeTypes.LLM, - node_run_result=original_result, - start_at=naive_utc_now(), - error="oops", - ) + def test_execute_node_safely_should_apply_error_strategy_on_failed_status(self, service: WorkflowService) -> None: + # Arrange + node = MagicMock() + node.error_strategy = ErrorStrategy.FAIL_BRANCH + node.default_value_dict = {} - def invoke_fn(): - def _gen(): - yield failed_event + original_result = NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + error="oops", + error_type="ValueError", + inputs={}, + ) + failed_event = NodeRunFailedEvent( + id=str(uuid.uuid4()), + node_id="node-1", + node_type=BuiltinNodeTypes.LLM, + node_run_result=original_result, + start_at=naive_utc_now(), + error="oops", + ) - return node, _gen() + def invoke_fn(): + def _gen(): + yield failed_event - # Act - _, result, run_succeeded, _ = service._execute_node_safely(invoke_fn) + return node, _gen() - # Assert — after applying error strategy status becomes EXCEPTION - assert result is not None - assert result.status == WorkflowNodeExecutionStatus.EXCEPTION - # run_succeeded should be True because EXCEPTION is in the succeeded set - assert run_succeeded is True + # Act + _, result, run_succeeded, _ = service._execute_node_safely(invoke_fn) - # =========================================================================== - # TestWorkflowServiceGetNodeLastRun - # Tests for get_node_last_run delegation to repository - # =========================================================================== + # Assert — after applying error strategy status becomes EXCEPTION + assert result is not None + assert result.status == WorkflowNodeExecutionStatus.EXCEPTION + # run_succeeded should be True because EXCEPTION is in the succeeded set + assert run_succeeded is True - class TestWorkflowServiceGetNodeLastRun: - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - def test_get_node_last_run_should_delegate_to_repository(self, service: WorkflowService) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id="wf-1") - expected = MagicMock() +# =========================================================================== +# TestWorkflowServiceGetNodeLastRun +# Tests for get_node_last_run delegation to repository +# =========================================================================== - service._node_execution_service_repo = MagicMock() - service._node_execution_service_repo.get_node_last_execution.return_value = expected - # Act - result = service.get_node_last_run(app, workflow, "node-42") +class TestWorkflowServiceGetNodeLastRun: + @pytest.fixture + def service(self, sqlite_engine: Engine) -> WorkflowService: + return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) + + def test_get_node_last_run_should_delegate_to_repository(self, service: WorkflowService) -> None: + # Arrange + app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") + workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id="wf-1") + expected = MagicMock() + + service._node_execution_service_repo = MagicMock() + service._node_execution_service_repo.get_node_last_execution.return_value = expected + + # Act + result = service.get_node_last_run(app, workflow, "node-42") + + # Assert + assert result is expected + service._node_execution_service_repo.get_node_last_execution.assert_called_once_with( + tenant_id="tenant-1", + app_id="app-1", + workflow_id="wf-1", + node_id="node-42", + ) - # Assert - assert result is expected - service._node_execution_service_repo.get_node_last_execution.assert_called_once_with( - tenant_id="tenant-1", - app_id="app-1", - workflow_id="wf-1", - node_id="node-42", - ) + def test_get_node_last_run_should_return_none_when_repository_returns_none(self, service: WorkflowService) -> None: + # Arrange + app = TestWorkflowAssociatedDataFactory.create_app(app_id="a", tenant_id="t") + workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id="w") + service._node_execution_service_repo = MagicMock() + service._node_execution_service_repo.get_node_last_execution.return_value = None - def test_get_node_last_run_should_return_none_when_repository_returns_none( - self, service: WorkflowService - ) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="a", tenant_id="t") - workflow = TestWorkflowAssociatedDataFactory.create_workflow(workflow_id="w") - service._node_execution_service_repo = MagicMock() - service._node_execution_service_repo.get_node_last_execution.return_value = None + # Act + result = service.get_node_last_run(app, workflow, "node-x") - # Act - result = service.get_node_last_run(app, workflow, "node-x") + # Assert + assert result is None - # Assert - assert result is None - # =========================================================================== - # TestWorkflowServiceModuleLevelHelpers - # Tests for module-level helper functions exported from workflow_service - # =========================================================================== +# =========================================================================== +# TestWorkflowServiceModuleLevelHelpers +# Tests for module-level helper functions exported from workflow_service +# =========================================================================== - class TestSetupVariablePool: - """ - Tests for the module-level `_setup_variable_pool` function. - This helper initialises the VariablePool used for single-step workflow execution. - """ - def _make_workflow(self, workflow_type: str = WorkflowType.WORKFLOW) -> Workflow: - return TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - app_id="app-1", - workflow_type=WorkflowType(workflow_type), - ) +class TestSetupVariablePool: + """ + Tests for the module-level `_setup_variable_pool` function. + This helper initialises the VariablePool used for single-step workflow execution. + """ - def test_setup_variable_pool_should_use_full_system_variables_for_start_node( - self, - ) -> None: - # Arrange - workflow = self._make_workflow() + def _make_workflow(self, workflow_type: str = WorkflowType.WORKFLOW) -> Workflow: + return TestWorkflowAssociatedDataFactory.create_workflow( + workflow_id="wf-1", + app_id="app-1", + workflow_type=WorkflowType(workflow_type), + ) - # Act - with ( - patch("services.workflow_service.VariablePool") as mock_pool, - patch("services.workflow_service.build_system_variables") as mock_build_system_variables, - patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, - patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, - patch("services.workflow_service.add_node_inputs_to_pool") as mock_add_node_inputs_to_pool, - ): - _setup_variable_pool( - query="hello", - files=[], - user_id="u-1", - user_inputs={"k": "v"}, - workflow=workflow, - node_id="start-node", - node_type=BuiltinNodeTypes.START, - conversation_id="conv-1", - conversation_variables=[], - ) + def test_setup_variable_pool_should_use_full_system_variables_for_start_node( + self, + ) -> None: + # Arrange + workflow = self._make_workflow() - # Assert — start nodes should build bootstrap variables and attach node inputs. - mock_pool.assert_called_once_with() - mock_build_system_variables.assert_called_once() - mock_add_variables_to_pool.assert_called_once_with( - mock_pool.return_value, - mock_build_bootstrap_variables.return_value, - ) - mock_add_node_inputs_to_pool.assert_called_once_with( - mock_pool.return_value, + # Act + with ( + patch("services.workflow_service.VariablePool") as mock_pool, + patch("services.workflow_service.build_system_variables") as mock_build_system_variables, + patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, + patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, + patch("services.workflow_service.add_node_inputs_to_pool") as mock_add_node_inputs_to_pool, + ): + _setup_variable_pool( + query="hello", + files=[], + user_id="u-1", + user_inputs={"k": "v"}, + workflow=workflow, node_id="start-node", - inputs={"k": "v"}, + node_type=BuiltinNodeTypes.START, + conversation_id="conv-1", + conversation_variables=[], ) - def test_setup_variable_pool_should_read_the_timestamp_from_the_epoch( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """sys.timestamp is a Unix timestamp, so it must not follow the host timezone.""" - if not hasattr(time, "tzset"): - pytest.skip("tzset is only available on POSIX platforms") - - recorded_at = datetime(2024, 5, 1, tzinfo=UTC) - previous_tz = os.environ.get("TZ") - monkeypatch.setenv("TZ", "Asia/Shanghai") - time.tzset() - try: - workflow = self._make_workflow() - monkeypatch.setattr("services.workflow_service.naive_utc_now", lambda: recorded_at.replace(tzinfo=None)) - - with ( - patch("services.workflow_service.VariablePool"), - patch("services.workflow_service.build_system_variables") as mock_build_system_variables, - patch("services.workflow_service.build_bootstrap_variables"), - patch("services.workflow_service.add_variables_to_pool"), - patch("services.workflow_service.add_node_inputs_to_pool"), - ): - _setup_variable_pool( - query="hello", - files=[], - user_id="u-1", - user_inputs={}, - workflow=workflow, - node_id="start-node", - node_type=BuiltinNodeTypes.START, - conversation_id="conv-1", - conversation_variables=[], - ) - finally: - if previous_tz is None: - monkeypatch.delenv("TZ", raising=False) - else: - monkeypatch.setenv("TZ", previous_tz) - time.tzset() - - values = mock_build_system_variables.call_args.args[0] - assert values["timestamp"] == int(recorded_at.timestamp()) - - def test_setup_variable_pool_should_use_default_system_variables_for_non_start_node( - self, - ) -> None: - # Arrange - workflow = self._make_workflow() + # Assert — start nodes should build bootstrap variables and attach node inputs. + mock_pool.assert_called_once_with() + mock_build_system_variables.assert_called_once() + mock_add_variables_to_pool.assert_called_once_with( + mock_pool.return_value, + mock_build_bootstrap_variables.return_value, + ) + mock_add_node_inputs_to_pool.assert_called_once_with( + mock_pool.return_value, + node_id="start-node", + inputs={"k": "v"}, + ) - # Act - with ( - patch("services.workflow_service.VariablePool") as mock_pool, - patch("services.workflow_service.default_system_variables") as mock_default_system_variables, - patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, - patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, - patch("services.workflow_service.add_node_inputs_to_pool") as mock_add_node_inputs_to_pool, - ): - _setup_variable_pool( - query="", - files=[], - user_id="u-1", - user_inputs={}, - workflow=workflow, - node_id="llm-node", - node_type=BuiltinNodeTypes.LLM, # not a start/trigger node - conversation_id="conv-1", - conversation_variables=[], - ) + def test_setup_variable_pool_should_use_default_system_variables_for_non_start_node( + self, + ) -> None: + # Arrange + workflow = self._make_workflow() - # Assert — default system variables should be used and node inputs should not be added. - mock_default_system_variables.assert_called_once() - mock_pool.assert_called_once_with() - mock_add_variables_to_pool.assert_called_once_with( - mock_pool.return_value, - mock_build_bootstrap_variables.return_value, + # Act + with ( + patch("services.workflow_service.VariablePool") as mock_pool, + patch("services.workflow_service.default_system_variables") as mock_default_system_variables, + patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, + patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, + patch("services.workflow_service.add_node_inputs_to_pool") as mock_add_node_inputs_to_pool, + ): + _setup_variable_pool( + query="", + files=[], + user_id="u-1", + user_inputs={}, + workflow=workflow, + node_id="llm-node", + node_type=BuiltinNodeTypes.LLM, # not a start/trigger node + conversation_id="conv-1", + conversation_variables=[], ) - mock_add_node_inputs_to_pool.assert_not_called() - - def test_setup_variable_pool_should_set_chatflow_specifics_for_non_workflow_type( - self, - ) -> None: - """For ADVANCED_CHAT workflows on a START node, query/conversation_id/dialogue_count should be set.""" - from models.workflow import WorkflowType - - # Arrange - workflow = self._make_workflow(workflow_type=WorkflowType.CHAT) - - # Act - with ( - patch("services.workflow_service.VariablePool") as mock_pool, - patch("services.workflow_service.build_system_variables") as mock_build_system_variables, - patch("services.workflow_service.build_bootstrap_variables"), - patch("services.workflow_service.add_variables_to_pool"), - patch("services.workflow_service.add_node_inputs_to_pool"), - ): - _setup_variable_pool( - query="what is AI?", - files=[], - user_id="u-1", - user_inputs={}, - workflow=workflow, - node_id="start-node", - node_type=BuiltinNodeTypes.START, - conversation_id="conv-abc", - conversation_variables=[], - ) - - # Assert — chatflow system variables should include query, conversation_id and dialogue_count. - mock_pool.assert_called_once_with() - system_variable_values = mock_build_system_variables.call_args.args[0] - assert system_variable_values["query"] == "what is AI?" - assert system_variable_values["conversation_id"] == "conv-abc" - assert system_variable_values["dialogue_count"] == 1 - - class TestRebuildSingleFile: - """ - Tests for the module-level `_rebuild_single_file` function. - Ensures correct delegation to `build_from_mapping` / `build_from_mappings`. - """ - - def test_rebuild_single_file_should_call_build_from_mapping_for_file_type( - self, - ) -> None: - # Arrange - tenant_id = "tenant-1" - value = {"url": "https://example.com/file.pdf", "type": "document"} - mock_file = MagicMock() - - # Act - with patch("services.workflow_service.build_from_mapping", return_value=mock_file) as mock_build: - result = _rebuild_single_file(tenant_id, value, VariableEntityType.FILE) - - # Assert - assert result is mock_file - mock_build.assert_called_once_with(mapping=value, tenant_id=tenant_id, access_controller=ANY) - - def test_rebuild_single_file_should_raise_when_file_value_not_dict( - self, - ) -> None: - # Arrange + Act + Assert - with pytest.raises(ValueError, match="expected dict for file object"): - _rebuild_single_file("tenant-1", "not-a-dict", VariableEntityType.FILE) - - def test_rebuild_single_file_should_call_build_from_mappings_for_file_list( - self, - ) -> None: - # Arrange - tenant_id = "tenant-1" - value = [{"url": "https://example.com/a.pdf"}, {"url": "https://example.com/b.pdf"}] - mock_files = [MagicMock(), MagicMock()] - - # Act - with patch("services.workflow_service.build_from_mappings", return_value=mock_files) as mock_build: - result = _rebuild_single_file(tenant_id, value, VariableEntityType.FILE_LIST) - - # Assert - assert result is mock_files - mock_build.assert_called_once_with(mappings=value, tenant_id=tenant_id, access_controller=ANY) - - def test_rebuild_single_file_should_raise_when_file_list_value_not_list( - self, - ) -> None: - # Arrange + Act + Assert - with pytest.raises(ValueError, match="expected list for file list object"): - _rebuild_single_file("tenant-1", "not-a-list", VariableEntityType.FILE_LIST) - - def test_rebuild_single_file_should_return_empty_list_for_empty_file_list( - self, - ) -> None: - # Arrange + Act - result = _rebuild_single_file("tenant-1", [], VariableEntityType.FILE_LIST) - - # Assert - assert result == [] - def test_rebuild_single_file_should_raise_when_first_element_not_dict( - self, - ) -> None: - # Arrange + Act + Assert - with pytest.raises(ValueError, match="expected dict for first element"): - _rebuild_single_file("tenant-1", ["not-a-dict"], VariableEntityType.FILE_LIST) + # Assert — default system variables should be used and node inputs should not be added. + mock_default_system_variables.assert_called_once() + mock_pool.assert_called_once_with() + mock_add_variables_to_pool.assert_called_once_with( + mock_pool.return_value, + mock_build_bootstrap_variables.return_value, + ) + mock_add_node_inputs_to_pool.assert_not_called() - class TestRebuildFileForUserInputsInStartNode: - """ - Tests for the module-level `_rebuild_file_for_user_inputs_in_start_node` function. - """ + def test_setup_variable_pool_should_set_chatflow_specifics_for_non_workflow_type( + self, + ) -> None: + """For ADVANCED_CHAT workflows on a START node, query/conversation_id/dialogue_count should be set.""" + from models.workflow import WorkflowType - def _make_start_node_data(self, variables: list) -> MagicMock: - start_data = MagicMock() - start_data.variables = variables - return start_data - - def _make_variable(self, name: str, var_type: VariableEntityType) -> MagicMock: - var = MagicMock() - var.variable = name - var.type = var_type - return var - - def test_rebuild_should_pass_through_non_file_variables( - self, - ) -> None: - # Arrange - text_var = self._make_variable("query", VariableEntityType.TEXT_INPUT) - start_data = self._make_start_node_data([text_var]) - user_inputs = {"query": "hello world"} + # Arrange + workflow = self._make_workflow(workflow_type=WorkflowType.CHAT) - # Act - result = _rebuild_file_for_user_inputs_in_start_node( - tenant_id="tenant-1", - start_node_data=start_data, - user_inputs=user_inputs, + # Act + with ( + patch("services.workflow_service.VariablePool") as mock_pool, + patch("services.workflow_service.build_system_variables") as mock_build_system_variables, + patch("services.workflow_service.build_bootstrap_variables"), + patch("services.workflow_service.add_variables_to_pool"), + patch("services.workflow_service.add_node_inputs_to_pool"), + ): + _setup_variable_pool( + query="what is AI?", + files=[], + user_id="u-1", + user_inputs={}, + workflow=workflow, + node_id="start-node", + node_type=BuiltinNodeTypes.START, + conversation_id="conv-abc", + conversation_variables=[], ) - # Assert — non-file inputs are untouched - assert result["query"] == "hello world" + # Assert — chatflow system variables should include query, conversation_id and dialogue_count. + mock_pool.assert_called_once_with() + system_variable_values = mock_build_system_variables.call_args.args[0] + assert system_variable_values["query"] == "what is AI?" + assert system_variable_values["conversation_id"] == "conv-abc" + assert system_variable_values["dialogue_count"] == 1 - def test_rebuild_should_rebuild_file_variable( - self, - ) -> None: - # Arrange - file_var = self._make_variable("attachment", VariableEntityType.FILE) - start_data = self._make_start_node_data([file_var]) - file_value = {"url": "https://example.com/file.pdf"} - user_inputs = {"attachment": file_value} - mock_file = MagicMock() - # Act - with patch("services.workflow_service.build_from_mapping", return_value=mock_file): - result = _rebuild_file_for_user_inputs_in_start_node( - tenant_id="tenant-1", - start_node_data=start_data, - user_inputs=user_inputs, - ) - - # Assert — the dict value should be replaced by the rebuilt File object - assert result["attachment"] is mock_file - - def test_rebuild_should_skip_variable_not_in_inputs( - self, - ) -> None: - # Arrange - file_var = self._make_variable("attachment", VariableEntityType.FILE) - start_data = self._make_start_node_data([file_var]) - user_inputs: dict[str, Any] = {} # attachment not provided +class TestRebuildSingleFile: + """ + Tests for the module-level `_rebuild_single_file` function. + Ensures correct delegation to `build_from_mapping` / `build_from_mappings`. + """ - # Act - result = _rebuild_file_for_user_inputs_in_start_node( - tenant_id="tenant-1", - start_node_data=start_data, - user_inputs=user_inputs, - ) + def test_rebuild_single_file_should_call_build_from_mapping_for_file_type( + self, + ) -> None: + # Arrange + tenant_id = "tenant-1" + value = {"url": "https://example.com/file.pdf", "type": "document"} + mock_file = MagicMock() - # Assert — no key should be added for missing inputs - assert "attachment" not in result + # Act + with patch("services.workflow_service.build_from_mapping", return_value=mock_file) as mock_build: + result = _rebuild_single_file(tenant_id, value, VariableEntityType.FILE) - class TestWorkflowServiceResolveDeliveryMethod: - """ - Tests for the static helper `_resolve_human_input_delivery_method`. - """ + # Assert + assert result is mock_file + mock_build.assert_called_once_with(mapping=value, tenant_id=tenant_id, access_controller=ANY) - def _make_method(self, method_id) -> MagicMock: - m = MagicMock() - m.id = method_id - return m + def test_rebuild_single_file_should_raise_when_file_value_not_dict( + self, + ) -> None: + # Arrange + Act + Assert + with pytest.raises(ValueError, match="expected dict for file object"): + _rebuild_single_file("tenant-1", "not-a-dict", VariableEntityType.FILE) - def test_resolve_delivery_method_should_return_method_when_id_matches(self) -> None: - # Arrange - method_a = self._make_method("method-1") - method_b = self._make_method("method-2") + def test_rebuild_single_file_should_call_build_from_mappings_for_file_list( + self, + ) -> None: + # Arrange + tenant_id = "tenant-1" + value = [{"url": "https://example.com/a.pdf"}, {"url": "https://example.com/b.pdf"}] + mock_files = [MagicMock(), MagicMock()] - # Act - with patch( - "services.workflow_service.parse_human_input_delivery_methods", return_value=[method_a, method_b] - ): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="method-2" - ) + # Act + with patch("services.workflow_service.build_from_mappings", return_value=mock_files) as mock_build: + result = _rebuild_single_file(tenant_id, value, VariableEntityType.FILE_LIST) - # Assert - assert result is method_b + # Assert + assert result is mock_files + mock_build.assert_called_once_with(mappings=value, tenant_id=tenant_id, access_controller=ANY) - def test_resolve_delivery_method_should_return_none_when_no_match(self) -> None: - # Arrange - method_a = self._make_method("method-1") + def test_rebuild_single_file_should_raise_when_file_list_value_not_list( + self, + ) -> None: + # Arrange + Act + Assert + with pytest.raises(ValueError, match="expected list for file list object"): + _rebuild_single_file("tenant-1", "not-a-list", VariableEntityType.FILE_LIST) - # Act - with patch("services.workflow_service.parse_human_input_delivery_methods", return_value=[method_a]): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="does-not-exist" - ) + def test_rebuild_single_file_should_return_empty_list_for_empty_file_list( + self, + ) -> None: + # Arrange + Act + result = _rebuild_single_file("tenant-1", [], VariableEntityType.FILE_LIST) - # Assert - assert result is None + # Assert + assert result == [] - def test_resolve_delivery_method_should_return_none_for_empty_methods(self) -> None: - # Act - with patch("services.workflow_service.parse_human_input_delivery_methods", return_value=[]): - result = WorkflowService._resolve_human_input_delivery_method( - node_data=MagicMock(), delivery_method_id="method-1" - ) + def test_rebuild_single_file_should_raise_when_first_element_not_dict( + self, + ) -> None: + # Arrange + Act + Assert + with pytest.raises(ValueError, match="expected dict for first element"): + _rebuild_single_file("tenant-1", ["not-a-dict"], VariableEntityType.FILE_LIST) - # Assert - assert result is None - - # =========================================================================== - # TestWorkflowServiceDraftExecution - # Tests for run_draft_workflow_node - # =========================================================================== - - class TestWorkflowServiceDraftExecution: - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - def test_run_draft_workflow_node_should_execute_start_node_successfully( - self, service: WorkflowService, sqlite_engine: _workflow_service_Engine - ) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - node_id = "start-node" - draft_workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": node_id, - "data": {"type": BuiltinNodeTypes.START, "title": "Start", "variables": []}, - } - ], - "edges": [], - }, - ) - # Mocking complex dependencies - with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.StartNodeData"), - patch( - "services.workflow_service._rebuild_file_for_user_inputs_in_start_node", - side_effect=lambda **kwargs: kwargs["user_inputs"], - ), - patch("services.workflow_service._setup_variable_pool"), - patch("services.workflow_service.DraftVarLoader"), - patch("services.workflow_service.WorkflowEntry.single_step_run") as mock_run, - patch("services.workflow_service.DifyCoreRepositoryFactory") as mock_repo_factory, - patch("services.workflow_service.DraftVariableSaver") as mock_saver_cls, - patch("services.workflow_service.storage"), - ): - mock_node = MagicMock() - mock_node.node_type = BuiltinNodeTypes.START - mock_node.title = "Start Node" - mock_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs={}, outputs={"result": "ok"} - ) - mock_event = NodeRunSucceededEvent( - id=str(uuid.uuid4()), - node_id="start-node", - node_type=BuiltinNodeTypes.START, - node_run_result=mock_run_result, - start_at=naive_utc_now(), - ) - mock_run.return_value = (mock_node, [mock_event]) +class TestRebuildFileForUserInputsInStartNode: + """ + Tests for the module-level `_rebuild_file_for_user_inputs_in_start_node` function. + """ - mock_repo = MagicMock() - mock_repo_factory.create_workflow_node_execution_repository.return_value = mock_repo + def _make_start_node_data(self, variables: list) -> MagicMock: + start_data = MagicMock() + start_data.variables = variables + return start_data - service._node_execution_service_repo = MagicMock() - mock_execution_record = MagicMock() - mock_execution_record.node_type = "start" - mock_execution_record.node_id = "start-node" - mock_execution_record.load_full_outputs.return_value = {} - service._node_execution_service_repo.get_execution_by_id.return_value = mock_execution_record + def _make_variable(self, name: str, var_type: VariableEntityType) -> MagicMock: + var = MagicMock() + var.variable = name + var.type = var_type + return var - # Act - result = service.run_draft_workflow_node( - app_model=app, - draft_workflow=draft_workflow, - account=account, - node_id=node_id, - user_inputs={"key": "val"}, - query="hi", - files=[], - ) + def test_rebuild_should_pass_through_non_file_variables( + self, + ) -> None: + # Arrange + text_var = self._make_variable("query", VariableEntityType.TEXT_INPUT) + start_data = self._make_start_node_data([text_var]) + user_inputs = {"query": "hello world"} + + # Act + result = _rebuild_file_for_user_inputs_in_start_node( + tenant_id="tenant-1", + start_node_data=start_data, + user_inputs=user_inputs, + ) + + # Assert — non-file inputs are untouched + assert result["query"] == "hello world" - # Assert - assert result is not None - mock_run.assert_called_once() - mock_repo.save_synchronously.assert_called_once() - mock_saver_cls.return_value.save.assert_called_once() - - def test_run_draft_workflow_node_should_execute_non_start_node_successfully( - self, service: WorkflowService, sqlite_engine: _workflow_service_Engine - ) -> None: - # Arrange - app = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - node_id = "llm-node" - draft_workflow = TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", + def test_rebuild_should_rebuild_file_variable( + self, + ) -> None: + # Arrange + file_var = self._make_variable("attachment", VariableEntityType.FILE) + start_data = self._make_start_node_data([file_var]) + file_value = {"url": "https://example.com/file.pdf"} + user_inputs = {"attachment": file_value} + mock_file = MagicMock() + + # Act + with patch("services.workflow_service.build_from_mapping", return_value=mock_file): + result = _rebuild_file_for_user_inputs_in_start_node( tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": node_id, - "data": { - "type": BuiltinNodeTypes.LLM, - "title": "LLM", - "model": {"provider": "openai", "name": "gpt-4"}, - }, - } - ], - "edges": [], - }, + start_node_data=start_data, + user_inputs=user_inputs, ) - with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.VariablePool") as mock_pool_cls, - patch("services.workflow_service.build_system_variables") as mock_build_system_variables, - patch("services.workflow_service.build_bootstrap_variables") as mock_build_bootstrap_variables, - patch("services.workflow_service.add_variables_to_pool") as mock_add_variables_to_pool, - patch("services.workflow_service.DraftVarLoader"), - patch("services.workflow_service.WorkflowEntry.single_step_run") as mock_run, - patch("services.workflow_service.DifyCoreRepositoryFactory"), - patch("services.workflow_service.DraftVariableSaver"), - patch("services.workflow_service.storage"), - ): - mock_node = MagicMock() - mock_node.node_type = BuiltinNodeTypes.LLM - mock_node.title = "LLM Node" - mock_run_result = NodeRunResult( - status=WorkflowNodeExecutionStatus.SUCCEEDED, inputs={}, outputs={"result": "ok"} - ) - mock_event = NodeRunSucceededEvent( - id=str(uuid.uuid4()), - node_id="llm-node", - node_type=BuiltinNodeTypes.LLM, - node_run_result=mock_run_result, - start_at=naive_utc_now(), - ) - mock_run.return_value = (mock_node, [mock_event]) + # Assert — the dict value should be replaced by the rebuilt File object + assert result["attachment"] is mock_file - service._node_execution_service_repo = MagicMock() - mock_execution_record = MagicMock() - mock_execution_record.node_type = "llm" - mock_execution_record.node_id = "llm-node" - mock_execution_record.load_full_outputs.return_value = {"answer": "hello"} - service._node_execution_service_repo.get_execution_by_id.return_value = mock_execution_record + def test_rebuild_should_skip_variable_not_in_inputs( + self, + ) -> None: + # Arrange + file_var = self._make_variable("attachment", VariableEntityType.FILE) + start_data = self._make_start_node_data([file_var]) + user_inputs: dict[str, Any] = {} # attachment not provided + + # Act + result = _rebuild_file_for_user_inputs_in_start_node( + tenant_id="tenant-1", + start_node_data=start_data, + user_inputs=user_inputs, + ) - # Act - service.run_draft_workflow_node( - app_model=app, - draft_workflow=draft_workflow, - account=account, - node_id=node_id, - user_inputs={}, - query="", - files=None, - ) + # Assert — no key should be added for missing inputs + assert "attachment" not in result - # Assert - # For non-start nodes, bootstrap variables should be loaded into an empty pool. - mock_pool_cls.assert_called_once_with() - mock_build_system_variables.assert_called_once_with() - mock_build_bootstrap_variables.assert_called_once_with( - system_variables=mock_build_system_variables.return_value, - environment_variables=draft_workflow.environment_variables, - ) - mock_add_variables_to_pool.assert_called_once_with( - mock_pool_cls.return_value, mock_build_bootstrap_variables.return_value - ) - @pytest.mark.parametrize( - "repository_type", - [ - SQLAlchemyWorkflowNodeExecutionRepository, - CeleryWorkflowNodeExecutionRepository, - LogstoreWorkflowNodeExecutionRepository, - ], +class TestWorkflowServiceDraftExecution: + @pytest.mark.parametrize( + "repository_type", + [ + SQLAlchemyWorkflowNodeExecutionRepository, + CeleryWorkflowNodeExecutionRepository, + LogstoreWorkflowNodeExecutionRepository, + ], + ) + @pytest.mark.parametrize("status", [WorkflowNodeExecutionStatus.SUCCEEDED, WorkflowNodeExecutionStatus.FAILED]) + def test_single_step_persists_caller_before_body_and_keeps_workspace_on_completion( + self, + sqlite_engine: Engine, + config_overrides: Callable[..., None], + repository_type: type[SQLAlchemyWorkflowNodeExecutionRepository] + | type[CeleryWorkflowNodeExecutionRepository] + | type[LogstoreWorkflowNodeExecutionRepository], + status: WorkflowNodeExecutionStatus, + *, + workflow_runtime: WorkflowExecutionDependencies, + workflow_variables: WorkflowExecutionVariables, + ) -> None: + app = TestWorkflowAssociatedDataFactory.create_app() + account = TestWorkflowAssociatedDataFactory.create_account() + node_id = "agent-node" + workflow_model = TestWorkflowAssociatedDataFactory.create_workflow( + graph={ + "nodes": [{"id": node_id, "data": {"type": "agent", "version": "2", "title": "Agent"}}], + "edges": [], + } ) - @pytest.mark.parametrize("status", [WorkflowNodeExecutionStatus.SUCCEEDED, WorkflowNodeExecutionStatus.FAILED]) - def test_single_step_persists_caller_before_body_and_keeps_workspace_on_completion( - self, - service: WorkflowService, - sqlite_engine: _workflow_service_Engine, - config_overrides: Callable[..., None], - repository_type: type[SQLAlchemyWorkflowNodeExecutionRepository] - | type[CeleryWorkflowNodeExecutionRepository] - | type[LogstoreWorkflowNodeExecutionRepository], - status: WorkflowNodeExecutionStatus, - ) -> None: - app = TestWorkflowAssociatedDataFactory.create_app() - account = TestWorkflowAssociatedDataFactory.create_account() - node_id = "agent-node" - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - graph={ - "nodes": [{"id": node_id, "data": {"type": "agent", "version": "2", "title": "Agent"}}], - "edges": [], - } + workflow = workflow_snapshot(workflow_model) + execution_id = str(uuid.uuid4()) + workspace_binding_id = str(uuid.uuid4()) + workflow_binding_id = str(uuid.uuid4()) + started_at = naive_utc_now() + config_overrides(AGENT_BACKEND_USE_FAKE=True, LOGSTORE_DUAL_WRITE_ENABLED=False) + + logstore_records: list[dict[str, str]] = [] + logstore = MagicMock() + logstore.put_log.side_effect = lambda _name, record: logstore_records.append(dict(record)) + logstore.execute_sql.side_effect = lambda **_kwargs: logstore_records[-1:] + with ( + patch( + "extensions.logstore.repositories.logstore_workflow_node_execution_repository.AliyunLogStore", + return_value=logstore, + ), + patch( + "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore", + return_value=logstore, + ), + ): + repository = repository_type( + session_factory=sqlite_engine, + tenant_id=app.tenant_id, + user=account, + app_id=app.id, + triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, + ) + if repository_type is LogstoreWorkflowNodeExecutionRepository: + config_overrides( + API_WORKFLOW_NODE_EXECUTION_REPOSITORY=( + "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository." + "LogstoreAPIWorkflowNodeExecutionRepository" + ) + ) + + runtime = replace(workflow_runtime, node_writer=MagicMock(return_value=repository)) + service = WorkflowService( + sessionmaker(bind=sqlite_engine, expire_on_commit=False), + runtime=runtime, ) - execution_id = str(uuid.uuid4()) - workspace_binding_id = str(uuid.uuid4()) - workflow_binding_id = str(uuid.uuid4()) - started_at = naive_utc_now() - config_overrides(AGENT_BACKEND_USE_FAKE=True) node = DifyNodeFactory( init_params=build_test_graph_init_params( workflow_id=workflow.id, @@ -3537,46 +1814,14 @@ def test_single_step_persists_caller_before_body_and_keeps_workspace_on_completi user_id=account.id, ), runtime_state=RuntimeState(workflow_id=workflow.id, variable_pool=VariablePool(), start_at=0), + agent_binding_resolver=runtime.agent_bindings, + workflow_runtime=runtime, ).create_node( { "id": node_id, "data": {"type": "agent", "version": "2", "title": "Agent", "agent_node_kind": "dify_agent"}, } ) - config_overrides(LOGSTORE_DUAL_WRITE_ENABLED=False) - logstore_records: list[dict[str, str]] = [] - logstore = MagicMock() - logstore.put_log.side_effect = lambda _name, record: logstore_records.append(dict(record)) - logstore.execute_sql.side_effect = lambda **_kwargs: logstore_records[-1:] - with ( - patch( - "extensions.logstore.repositories.logstore_workflow_node_execution_repository.AliyunLogStore", - return_value=logstore, - ), - patch( - "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository.AliyunLogStore", - return_value=logstore, - ), - ): - repository = repository_type( - session_factory=sqlite_engine, - tenant_id=app.tenant_id, - user=account, - app_id=app.id, - triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP, - ) - if repository_type is LogstoreWorkflowNodeExecutionRepository: - config_overrides( - API_WORKFLOW_NODE_EXECUTION_REPOSITORY=( - "extensions.logstore.repositories.logstore_api_workflow_node_execution_repository." - "LogstoreAPIWorkflowNodeExecutionRepository" - ) - ) - service._node_execution_service_repo = ( - DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository( - session_maker=sessionmaker(sqlite_engine) - ) - ) def node_events(variable_pool: VariablePool) -> Generator[NodeEvent]: yield NodeRunStartedEvent( @@ -3624,927 +1869,180 @@ def node_events(variable_pool: VariablePool) -> Generator[NodeEvent]: yield NodeRunSucceededEvent(**event_kwargs) with ( - patch("services.workflow_service.db", SimpleNamespace(engine=sqlite_engine)), - patch("services.workflow_service.WorkflowDraftVariableService"), - patch("services.workflow_service.DraftVarLoader"), patch( "services.workflow_service.WorkflowEntry.single_step_run", side_effect=lambda **kwargs: (node, node_events(kwargs["variable_pool"])), ), - patch( - "services.workflow_service.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", - return_value=repository, - ), - patch("services.workflow_service.DraftVariableSaver"), patch("services.workflow_service.enqueue_draft_node_execution_trace"), patch( - "core.repositories.celery_workflow_node_execution_repository.save_workflow_node_execution_task.delay" + "core.repositories.celery_workflow_node_execution_repository." + "save_workflow_node_execution_task.delay" ) as async_save, ): result = service.run_draft_workflow_node( - app_model=app, draft_workflow=workflow, node_id=node_id, user_inputs={}, account=account - ) - - assert result.id == execution_id - assert result.node_execution_id == execution_id - assert result.status == status - assert result.created_at == started_at - assert result.finished_at is not None - if status == WorkflowNodeExecutionStatus.SUCCEEDED: - assert result.outputs_dict == {"text": "done"} - assert result.process_data_dict is not None - assert result.process_data_dict["step"] == "completed" - else: - assert result.error == "runtime failure" - with Session(sqlite_engine) as session: - assert list(session.scalars(select(WorkflowNodeExecutionModel.id))) == [execution_id] - persisted = session.get(WorkflowNodeExecutionModel, execution_id) - assert persisted is not None - assert persisted.status == status - assert persisted.agent_workspace_binding_id == workspace_binding_id - assert persisted.process_data_dict is not None - assert persisted.process_data_dict["workflow_agent_binding_id"] == workflow_binding_id - if repository_type is CeleryWorkflowNodeExecutionRepository: - async_save.assert_called_once() - assert async_save.call_args.kwargs["execution_data"]["status"] == status - else: - async_save.assert_not_called() - if repository_type is LogstoreWorkflowNodeExecutionRepository: - assert len(logstore_records) == 1 - assert logstore_records[0]["status"] == status - - # =========================================================================== - # TestWorkflowServiceHumanInputOperations - # Tests for Human Input related methods - # =========================================================================== - - class TestWorkflowServiceHumanInputOperations: - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - @staticmethod - def _create_human_input_workflow() -> Workflow: - return TestWorkflowAssociatedDataFactory.create_workflow( - workflow_id="wf-1", - tenant_id="tenant-1", - app_id="app-1", - graph={ - "nodes": [ - { - "id": "node-1", - "data": {"type": BuiltinNodeTypes.HUMAN_INPUT, "title": "Human Input"}, - } - ], - "edges": [], - }, - ) - - def test_get_human_input_form_preview_should_raise_if_workflow_not_init( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - service.get_draft_workflow = MagicMock(return_value=None) - with pytest.raises(ValueError, match="Workflow not initialized"): - service.get_human_input_form_preview( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - session=sqlite_session, - ) - - def test_get_human_input_form_preview_should_raise_if_wrong_node_type( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - draft = TestWorkflowAssociatedDataFactory.create_workflow( - graph={"nodes": [{"id": "node-1", "data": {"type": "llm", "title": "LLM"}}], "edges": []} - ) - service.get_draft_workflow = MagicMock(return_value=draft) - with pytest.raises(ValueError, match="Node type must be human-input"): - service.get_human_input_form_preview( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - session=sqlite_session, - ) - - def test_get_human_input_form_preview_success(self, service: WorkflowService, sqlite_session: Session) -> None: - app_model = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - mock_node = MagicMock() - mock_node.render_form_content_before_submission.return_value = "rendered" - mock_node.resolve_default_values.return_value = {"def": 1} - mock_node.title = "Form Title" - mock_node.node_data = MagicMock() - - with ( - patch.object(service, "_build_human_input_variable_pool"), - patch("services.workflow_service.HumanInputNode", return_value=mock_node), - patch("services.workflow_service.HumanInputRequired") as mock_required_cls, - ): - service.get_human_input_form_preview( - app_model=app_model, account=account, node_id="node-1", session=sqlite_session - ) - mock_node.render_form_content_before_submission.assert_called_once() - mock_required_cls.return_value.model_dump.assert_called_once() - - def test_submit_human_input_form_preview_success( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - app_model = TestWorkflowAssociatedDataFactory.create_app(app_id="app-1", tenant_id="tenant-1") - account = TestWorkflowAssociatedDataFactory.create_account(account_id="user-1") - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) - - mock_node = MagicMock() - mock_node.node_data = MagicMock() - mock_node.node_data.user_actions = [ - SimpleNamespace(id="submit", title="card_visa_enterprise_001"), - ] - mock_node.node_data.outputs_field_names.return_value = ["field1"] - mock_node.node_data.inputs = [] - mock_node.render_form_content_before_submission.return_value = "Ticket: {{#$output.field1#}}" - mock_node.render_form_content_with_outputs.return_value = "Ticket: val1" - - with ( - patch("services.workflow_service.db"), - patch.object(service, "_build_human_input_variable_pool"), - patch("services.workflow_service.HumanInputNode", return_value=mock_node), - patch( - "services.workflow_service.HumanInputService.validate_and_normalize_submission", - return_value={"field1": "val1"}, - ) as mock_validate, - patch("services.workflow_service.DraftVariableSaver") as mock_saver_cls, - ): - result = service.submit_human_input_form_preview( - app_model=app_model, + app_model=app, + draft_workflow=workflow, + node_id=node_id, + user_inputs={}, account=account, - node_id="node-1", - form_inputs={"field1": "val1"}, - action="submit", - session=sqlite_session, + variables=workflow_variables, ) - assert result["__action_id"] == "submit" - mock_validate.assert_called_once() - assert result["__action_value"] == "card_visa_enterprise_001" - assert result["__rendered_content"] == "Ticket: val1" - mock_saver_cls.return_value.save.assert_called_once() - def test_test_human_input_delivery_success(self, service: WorkflowService, sqlite_session: Session) -> None: - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) + assert result.id == execution_id + assert result.node_execution_id == execution_id + assert result.status == status + assert result.created_at == started_at + assert result.finished_at is not None + if status == WorkflowNodeExecutionStatus.SUCCEEDED: + assert result.outputs_dict == {"text": "done"} + assert result.process_data_dict is not None + assert result.process_data_dict["step"] == "completed" + else: + assert result.error == "runtime failure" + with Session(sqlite_engine) as session: + assert list(session.scalars(select(WorkflowNodeExecutionModel.id))) == [execution_id] + persisted = session.get(WorkflowNodeExecutionModel, execution_id) + assert persisted is not None + assert persisted.status == status + assert persisted.agent_workspace_binding_id == workspace_binding_id + assert persisted.process_data_dict is not None + assert persisted.process_data_dict["workflow_agent_binding_id"] == workflow_binding_id + if repository_type is CeleryWorkflowNodeExecutionRepository: + async_save.assert_called_once() + assert async_save.call_args.kwargs["execution_data"]["status"] == status + else: + async_save.assert_not_called() + if repository_type is LogstoreWorkflowNodeExecutionRepository: + assert len(logstore_records) == 1 + assert logstore_records[0]["status"] == status - with ( - patch("services.workflow_service.HumanInputNodeData.model_validate"), - patch.object(service, "_resolve_human_input_delivery_method") as mock_resolve, - patch("services.workflow_service.apply_dify_debug_email_recipient"), - patch.object(service, "_build_human_input_variable_pool"), - patch.object(service, "_build_human_input_node_for_debugging"), - patch.object(service, "_create_human_input_delivery_test_form", return_value=("form-1", [])), - patch("services.workflow_service.HumanInputDeliveryTestService") as mock_test_srv, - ): - mock_resolve.return_value = MagicMock() - service.test_human_input_delivery( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - delivery_method_id="method-1", - session=sqlite_session, - ) - mock_test_srv.return_value.send_test.assert_called_once() - def test_test_human_input_delivery_failure_cases( - self, service: WorkflowService, sqlite_session: Session - ) -> None: - draft = self._create_human_input_workflow() - service.get_draft_workflow = MagicMock(return_value=draft) +# =========================================================================== +# TestWorkflowServiceHumanInputOperations +# Tests for Human Input related methods +# =========================================================================== - with ( - patch("services.workflow_service.HumanInputNodeData.model_validate"), - patch.object(service, "_resolve_human_input_delivery_method", return_value=None), - ): - with pytest.raises(ValueError, match="Delivery method not found"): - service.test_human_input_delivery( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="node-1", - delivery_method_id="none", - session=sqlite_session, - ) - def test_load_email_recipients_parsing_failure(self, service: WorkflowService, sqlite_session: Session) -> None: - """Malformed persisted recipient payloads are skipped instead of aborting delivery tests.""" - recipient = HumanInputFormRecipient( - form_id="form-1", - delivery_id="delivery-1", - recipient_payload="invalid json", - recipient_type=RecipientType.EMAIL_MEMBER, - access_token="recipient-token", - ) - sqlite_session.add(recipient) - sqlite_session.commit() - - with patch("services.workflow_service.db") as mock_db: - mock_db.engine = sqlite_session.get_bind() - result = WorkflowService._load_email_recipients("form-1") - - assert result == [] - - def test_form_preview_resolves_nested_inputs_without_loading_email_dependencies( - self, - service: WorkflowService, - sqlite_engine: _workflow_service_Engine, - sqlite_session: Session, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - workflow = TestWorkflowAssociatedDataFactory.create_workflow( - graph={ - "nodes": [ - { - "id": "human", - "data": { - "type": "human-input", - "title": "Human", - "form_content": "Hello {{#source.object.name#}}", - "inputs": [ - { - "type": "paragraph", - "output_variable_name": "comment", - "default": {"type": "variable", "selector": ["source", "object", "default"]}, - } - ], - "delivery_methods": [ - { - "type": "email", - "enabled": True, - "config": { - "recipients": {"items": []}, - "subject": "Review", - "body": "{{#email.missing#}} {{#url#}}", - }, - } - ], +class TestHumanInputDebugServiceRegression: + def test_form_preview_resolves_nested_inputs_without_loading_email_dependencies( + self, + sqlite_session_factory: sessionmaker[Session], + ) -> None: + definitions = MagicMock() + definitions.debug_node.return_value = WorkflowDebugNode( + config=cast( + Any, + { + "id": "human", + "data": { + "type": "human-input", + "title": "Human", + "form_content": "Hello {{#source.object.name#}}", + "inputs": [ + { + "type": "paragraph", + "output_variable_name": "comment", + "default": { + "type": "variable", + "selector": ["source", "object", "default"], + }, }, - } - ], - "edges": [], - } - ) - monkeypatch.setattr(service, "get_draft_workflow", lambda **_: workflow) - monkeypatch.setattr(type(db), "engine", property(lambda _: sqlite_engine)) - result = service.get_human_input_form_preview( - app_model=TestWorkflowAssociatedDataFactory.create_app(), - account=TestWorkflowAssociatedDataFactory.create_account(), - node_id="human", - inputs={"#source.object.name#": "Alice", "#source.object.default#": "Approved"}, - session=sqlite_session, - ) - - assert result["form_content"] == "Hello Alice" - assert result["resolved_default_values"] == {"comment": "Approved"} - - # =========================================================================== - # TestWorkflowServiceFreeNodeExecution - # Tests for run_free_workflow_node and handle_single_step_result - # =========================================================================== - - class TestWorkflowServiceFreeNodeExecution: - @pytest.fixture - def service(self, sqlite_engine: _workflow_service_Engine) -> WorkflowService: - return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - - def test_run_free_workflow_node_success(self, service: WorkflowService) -> None: - node_execution = MagicMock() - with ( - patch.object(service, "_handle_single_step_result", return_value=node_execution), - patch("services.workflow_service.WorkflowEntry.run_free_node"), - ): - result = service.run_free_workflow_node({}, "tenant-1", "user-1", "node-1", {}) - assert result == node_execution - - def test_validate_graph_structure_coexist_error(self, service: WorkflowService) -> None: - graph = { - "nodes": [ - {"data": {"type": "start"}}, - {"data": {"type": "trigger-webhook"}}, # is_trigger_node=True - ] - } - with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): - service.validate_graph_structure(graph) - - def test_validate_features_structure_success(self, service: WorkflowService) -> None: - app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) - features = {} - with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_val: - service.validate_features_structure(app, features) - mock_val.assert_called_once() - - def test_validate_features_structure_invalid_mode(self, service: WorkflowService) -> None: - app = TestWorkflowAssociatedDataFactory.create_app() - app.mode = "invalid" - with pytest.raises(ValueError, match="Invalid app mode"): - service.validate_features_structure(app, {}) - - def test_validate_human_input_node_data_error(self, service: WorkflowService) -> None: - with patch("services.workflow_service.HumanInputNodeData.model_validate", side_effect=Exception("error")): - with pytest.raises(ValueError, match="Invalid HumanInput node data"): - service._validate_human_input_node_data({}) - - def test_rebuild_single_file_unreachable(self) -> None: - # Test line 1523 (unreachable) - with pytest.raises(Exception, match="unreachable"): - _rebuild_single_file("tenant-1", {}, cast(Any, "invalid_type")) - - def test_build_human_input_node_for_debugging(self, service: WorkflowService) -> None: - """Cover _build_human_input_node_for_debugging.""" - workflow = TestWorkflowAssociatedDataFactory.create_workflow() - account = TestWorkflowAssociatedDataFactory.create_account() - node_config = {"id": "n-1", "data": {"type": BuiltinNodeTypes.HUMAN_INPUT, "title": "Human Input"}} - variable_pool = MagicMock() - node_data = MagicMock() - node_data.title = "Human Input" - - with ( - patch( - "services.workflow_service.adapt_human_input_node_data_for_graph", - return_value=sentinel.adapted_node_data, - ) as mock_adapt_node_data, - patch("services.workflow_service.HumanInputNodeData.model_validate", return_value=node_data), - ): - node = service._build_human_input_node_for_debugging( - workflow=workflow, account=account, node_config=node_config, variable_pool=variable_pool - ) - mock_adapt_node_data.assert_called_once_with(node_config["data"]) - assert node.node_id == "n-1" - assert node.title == "Human Input" - assert node.node_data is node_data - assert node.variable_pool is variable_pool - - @staticmethod - @pytest.mark.parametrize( - "upsert", - [_env("new-id", "TAKEN"), _env("other-id", "TAKEN")], - ids=["new id reuses a name", "rename onto another variable's name"], - ) - def test_environment_patch_rejects_a_name_another_id_uses(upsert: StringVariable) -> None: - current = [_env("taken-id", "TAKEN"), _env("other-id", "OTHER")] - with pytest.raises(ValueError, match="Environment variable names must be unique"): - _merge_environment_variable_patch(current, [upsert], []) - - -"""Tests for human-input delivery with a persisted draft workflow.""" - - -TENANT_ID = "11111111-1111-1111-1111-111111111111" -APP_ID = "22222222-2222-2222-2222-222222222222" -ACCOUNT_ID = "33333333-3333-3333-3333-333333333333" - - -def _make_service(sqlite_session: Session) -> WorkflowService: - return WorkflowService( - session_maker=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False), - ) - - -def _app() -> App: - return make_app( - app_id=APP_ID, - tenant_id=TENANT_ID, - mode=AppMode.WORKFLOW, - icon_type=None, - enable_site=False, - enable_api=False, - ) - - -def _account() -> _workflow_human_input_delivery_Account: - return make_account(account_id=ACCOUNT_ID, name="Test User", email="test@example.com") - - -def _build_node_config(delivery_methods: list[EmailDeliveryMethod]) -> dict[str, object]: - return { - "id": "node-1", - "data": HumanInputNodeData( - title="Human Input", - type=BuiltinNodeTypes.HUMAN_INPUT, - delivery_methods=delivery_methods, - form_content="Test content", - inputs=[], - user_actions=[], - ), - } - - -def _persist_workflow(sqlite_session: Session, delivery_methods: list[EmailDeliveryMethod]) -> None: - node_config = _build_node_config(delivery_methods) - node_data = node_config["data"] - assert isinstance(node_data, HumanInputNodeData) - workflow = Workflow.new( - tenant_id=TENANT_ID, - app_id=APP_ID, - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph=json.dumps( - { - "nodes": [ - { - "id": node_config["id"], - "data": node_data.model_dump(mode="json"), - } - ], - "edges": [], - } - ), - features="{}", - created_by=ACCOUNT_ID, - environment_variables=[], - conversation_variables=[], - rag_pipeline_variables=[], - ) - sqlite_session.add(workflow) - sqlite_session.commit() - sqlite_session.expunge_all() - - -def _make_email_method(enabled: bool = True, debug_mode: bool = False) -> EmailDeliveryMethod: - return EmailDeliveryMethod( - id=uuid.uuid4(), - enabled=enabled, - config=EmailDeliveryConfig( - recipients=EmailRecipients( - include_bound_group=False, - items=[ExternalRecipient(email="tester@example.com")], + ], + "delivery_methods": [ + { + "type": "email", + "enabled": True, + "config": { + "recipients": {"items": []}, + "subject": "Review", + "body": "{{#email.missing#}} {{#url#}}", + }, + }, + ], + }, + }, ), - subject="Test subject", - body="Test body", - debug_mode=debug_mode, - ), - ) - - -class TestWorkflowHumanInputDeliveryCases: - @staticmethod - @pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) - def test_human_input_delivery_requires_draft_workflow(sqlite_session: Session): - service = _make_service(sqlite_session) - app_model = _app() - account = _account() - - with pytest.raises(ValueError, match="Workflow not initialized"): - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id="delivery-1", - session=sqlite_session, - ) - assert sqlite_session.in_transaction() - - @staticmethod - @pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) - def test_human_input_delivery_allows_disabled_method( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, - ): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=False) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) - ) - - test_service_instance = MagicMock() - monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), - ) - - app_model = _app() - account = _account() - - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - session=sqlite_session, - ) - - test_service_instance.send_test.assert_called_once() - assert sqlite_session.in_transaction() - - @staticmethod - @pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) - def test_human_input_delivery_dispatches_to_test_service( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, - ): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=True) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) - ) - - test_service_instance = MagicMock() - monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), - ) - - app_model = _app() - account = _account() - - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - inputs={"#node-1.output#": "value"}, - session=sqlite_session, + environment_variables="{}", + conversation_variables=[], + enclosing_node_id=None, ) - - pool_args = service._build_human_input_variable_pool.call_args.kwargs - assert pool_args["manual_inputs"] == {"#node-1.output#": "value"} - test_service_instance.send_test.assert_called_once() - assert sqlite_session.in_transaction() - - @staticmethod - @pytest.mark.parametrize("sqlite_session", [(Workflow,)], indirect=True) - def test_human_input_delivery_debug_mode_overrides_recipients( - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, - ): - service = _make_service(sqlite_session) - delivery_method = _make_email_method(enabled=True, debug_mode=True) - _persist_workflow(sqlite_session, [delivery_method]) - service._build_human_input_variable_pool = MagicMock(return_value=MagicMock()) # type: ignore[attr-defined] - node_stub = MagicMock() - node_stub.render_form_content_before_submission.return_value = "rendered" - node_stub.resolve_default_values.return_value = {} - service._build_human_input_node_for_debugging = MagicMock(return_value=node_stub) # type: ignore[attr-defined] - service._create_human_input_delivery_test_form = MagicMock( # type: ignore[attr-defined] - return_value=("form-1", []) + variables = MagicMock() + variables.loader.return_value.load_variables.return_value = [] + service = HumanInputDebugService( + definitions=definitions, + variables=variables, + files=HumanInputDebugFileGateway(sessions=sqlite_session_factory, access=MagicMock()), + submissions=MagicMock(), + forms=MagicMock(), + delivery=MagicMock(), ) - - test_service_instance = MagicMock() - monkeypatch.setattr( - workflow_service_module, - "HumanInputDeliveryTestService", - MagicMock(return_value=test_service_instance), + context = RequestContext( + request_id="request-1", + trace_id=None, + account_id="user-1", + active_workspace_id="tenant-1", ) - app_model = _app() - account = _account() - - service.test_human_input_delivery( - app_model=app_model, - account=account, - node_id="node-1", - delivery_method_id=str(delivery_method.id), - session=sqlite_session, + result = service.preview_form( + context, + "app-1", + "human", + {"#source.object.name#": "Alice", "#source.object.default#": "Approved"}, ) - test_service_instance.send_test.assert_called_once() - sent_method = test_service_instance.send_test.call_args.kwargs["method"] - assert isinstance(sent_method, EmailDeliveryMethod) - assert sent_method.config.debug_mode is True - assert sent_method.config.recipients.include_bound_group is False - assert len(sent_method.config.recipients.items) == 1 - recipient = sent_method.config.recipients.items[0] - assert isinstance(recipient, MemberRecipient) - assert recipient.reference_id == account.id - assert sqlite_session.in_transaction() - - -"""Agent preview callers are committed before execution and retain their identity.""" - + assert result["form_content"] == "Hello Alice" + assert result["resolved_default_values"] == {"comment": "Approved"} -Factory = TestWorkflowAssociatedDataFactory +# =========================================================================== +# TestWorkflowServiceFreeNodeExecution +# Tests for run_free_workflow_node and handle_single_step_result +# =========================================================================== -class PreviewCancelled(BaseException): - """Cancellation can bypass Exception handlers, like a stopped greenlet.""" - -class TestWorkflowAgentPreviewIdentityCases: - @staticmethod +class TestWorkflowServiceFreeNodeExecution: @pytest.fixture - def preview_flask_app( - sqlite_engine: _workflow_agent_preview_identity_Engine, monkeypatch: pytest.MonkeyPatch - ) -> Iterator[Flask]: - apply_config_overrides(monkeypatch, AGENT_BACKEND_BASE_URL="https://agent.test") - app = Flask(__name__) - app.config["SQLALCHEMY_DATABASE_URI"] = sqlite_engine.url.render_as_string(hide_password=False) - db.init_app(app) - with app.app_context(): - yield app - - @staticmethod - @pytest.mark.parametrize("outcome", ["success", "failure", "exception", "cancel"]) - @pytest.mark.usefixtures("preview_flask_app") - def test_agent_preview_commits_runtime_identity_and_updates_same_row( - sqlite_engine: _workflow_agent_preview_identity_Engine, outcome: str - ) -> None: - tenant_id, app_id, workflow_id, account_id = (str(uuid4()) for _ in range(4)) - node_id = "agent-preview" - app = Factory.create_app(app_id=app_id, tenant_id=tenant_id) - account = Factory.create_account(account_id=account_id) - workflow = Factory.create_workflow( - workflow_id=workflow_id, - tenant_id=tenant_id, - app_id=app_id, - graph={ - "nodes": [ - { - "id": node_id, - "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent", "title": "Agent"}, - } - ], - "edges": [], - }, - ) - sessions = sessionmaker(sqlite_engine, expire_on_commit=False) - with sessions.begin() as session: - session.add_all([app, account, workflow]) - repository = SQLAlchemyWorkflowNodeExecutionRepository( - sessions, tenant_id, account, app_id, WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP - ) - observed_created_at = [] - execution_ids: list[str] = [] - - def events(node: DifyAgentNode) -> Generator[NodeRunStartedEvent | NodeRunSucceededEvent | NodeRunFailedEvent]: - execution_id = node.execution_id - execution_ids.append(execution_id) - started_at = naive_utc_now() - # Node.run now emits its typed start before entering the node body. The - # service observer must commit that event before resuming this generator. - yield NodeRunStartedEvent( - id=execution_id, - node_id=node_id, - node_type="agent", - node_version="2", - node_title="Agent", - start_at=started_at, - ) - # A separate transaction must see the committed caller before any - # Agent workspace or tracing request can be dispatched. - variable_pool = node.runtime_state.variable_pool - workflow_run_id = get_system_text(variable_pool, SystemVariableKey.WORKFLOW_EXECUTION_ID) - assert workflow_run_id is None - with sessions.begin() as session: - caller = WorkflowAgentWorkspaceStore._load_execution_by_identity( - session=session, - tenant_id=tenant_id, - app_id=app_id, - workflow_id=workflow_id, - workflow_run_id=workflow_run_id, - node_id=node_id, - node_execution_id=execution_id, - ) - assert caller.id == caller.node_execution_id == execution_id - assert caller.status == WorkflowNodeExecutionStatus.RUNNING - assert caller.finished_at is None - observed_created_at.append(caller.created_at) - # Session materialization writes these independently; final persistence - # must not discard either piece of caller ownership. - caller.agent_workspace_binding_id = "workspace-binding" - caller.process_data = json.dumps({"workflow_agent_binding_id": "workflow-binding"}) - if outcome == "exception": - raise RuntimeError("private runtime error details") - if outcome == "cancel": - raise PreviewCancelled() - status = ( - WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED - ) - result = NodeRunResult( - status=status, outputs={"answer": "done"}, error="" if outcome == "success" else "failed" - ) - if outcome == "success": - yield NodeRunSucceededEvent( - id=execution_id, node_id=node_id, node_type="agent", node_run_result=result, start_at=started_at - ) - else: - yield NodeRunFailedEvent( - id=execution_id, - node_id=node_id, - node_type="agent", - node_run_result=result, - start_at=started_at, - error="failed", - ) - - service = WorkflowService(sessions) - with ( - patch( - "services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values" - ), - patch.object(DifyAgentNode, "run", events), - patch( - "services.workflow_service.DifyCoreRepositoryFactory.create_workflow_node_execution_repository", - return_value=repository, - ), - patch("services.workflow_service.DraftVariableSaver.save"), - patch("services.workflow_service.enqueue_draft_node_execution_trace"), - ): - if outcome in {"exception", "cancel"}: - with pytest.raises(RuntimeError if outcome == "exception" else PreviewCancelled): - service.run_draft_workflow_node(app, workflow, node_id, {}, account) - else: - response = service.run_draft_workflow_node(app, workflow, node_id, {}, account) - assert response.id == response.node_execution_id == execution_ids[0] - - assert len(execution_ids) == 1 - execution_id = execution_ids[0] - with sessions() as session: - assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 1 - row = session.get(WorkflowNodeExecutionModel, execution_id) - assert row is not None - assert row.id == row.node_execution_id == execution_id - assert row.workflow_run_id is None - assert row.workflow_id == workflow_id - assert row.tenant_id == tenant_id - assert row.app_id == app_id - assert row.created_at == observed_created_at[0] - assert row.finished_at is not None - assert row.elapsed_time >= 0 - assert row.agent_workspace_binding_id == "workspace-binding" - assert row.process_data_dict is not None - assert row.process_data_dict["workflow_agent_binding_id"] == "workflow-binding" - assert row.status == ( - WorkflowNodeExecutionStatus.SUCCEEDED if outcome == "success" else WorkflowNodeExecutionStatus.FAILED - ) - if outcome in {"exception", "cancel"}: - assert row.error == "Agent preview execution interrupted" + def service(self, sqlite_engine: Engine) -> WorkflowService: + return WorkflowService(sessionmaker(bind=sqlite_engine, expire_on_commit=False)) - @staticmethod - @pytest.mark.usefixtures("preview_flask_app") - def test_agent_preview_rejects_mismatched_workflow_before_creating_caller( - sqlite_engine: _workflow_agent_preview_identity_Engine, - ) -> None: - app = Factory.create_app(app_id="app", tenant_id="tenant") - account = Factory.create_account() - workflow = Factory.create_workflow( - app_id="another-app", - tenant_id="another-tenant", - graph={ - "nodes": [{"id": "agent", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}], - "edges": [], - }, - ) - sessions = sessionmaker(sqlite_engine, expire_on_commit=False) + def test_run_free_workflow_node_success(self, service: WorkflowService) -> None: + node_execution = MagicMock() with ( - patch( - "services.workflow_service.WorkflowDraftVariableService.prefill_conversation_variable_default_values" - ) as prefill, - pytest.raises(ValueError, match="does not belong"), + patch.object(service, "_handle_single_step_result", return_value=node_execution), + patch("services.workflow_service.WorkflowEntry.run_free_node"), ): - WorkflowService(sessions).run_draft_workflow_node(app, workflow, "agent", {}, account) - prefill.assert_not_called() - with sessions() as session: - assert session.scalar(select(func.count()).select_from(WorkflowNodeExecutionModel)) == 0 - - -@pytest.mark.parametrize("source_mode", [AppMode.CHAT, AppMode.COMPLETION]) -@pytest.mark.parametrize( - ("access_mode", "outcome"), - [ - ("public", "success"), - ("private", "success"), - ("private_all", "success"), - ("sso_verified", "success"), - ("private", "read_timeout"), - ("private", "write_error"), - ("private", "false_result"), - ("private", "disabled"), - ("private", "conversion_error"), - ], -) -def test_convert_to_workflow_inherits_webapp_access( - sqlite_session_factory: sessionmaker[Session], - monkeypatch: pytest.MonkeyPatch, - source_mode: AppMode, - access_mode: str, - outcome: str, -) -> None: - import httpx - from blinker import Signal - - from events.event_handlers.create_installed_app_when_app_created import handle as create_installation - from events.event_handlers.create_site_record_when_app_created import handle as create_site - from models.model import AppModelConfig, InstalledApp, Site - from services.enterprise.enterprise_service import EnterpriseRequest - from services.errors.enterprise import EnterpriseServiceError - from services.workflow import workflow_converter as converter_module - - apply_config_overrides( - monkeypatch, - DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY if outcome == "disabled" else DeploymentEdition.ENTERPRISE, - ) - created_signal = Signal() - created_signal.connect(create_site) - created_signal.connect(create_installation) - monkeypatch.setattr(converter_module, "app_was_created", created_signal) - source_id = str(uuid4()) - calls: list[str] = [] - written: dict[str, str] = {} - write_error = EnterpriseServiceError("write failed") - - def build_graph( - _converter: converter_module.WorkflowConverter, - *, - app_model: App, - app_model_config: AppModelConfig, - account_id: str, - session: Session, - ) -> Workflow: - assert app_model_config.app_id == app_model.id - if outcome == "conversion_error": - raise ValueError("conversion failed") - workflow = Workflow( - tenant_id=app_model.tenant_id, - app_id=app_model.id, - type=WorkflowType.CHAT if source_mode == AppMode.CHAT else WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph='{"nodes": [], "edges": []}', - _features="{}", - created_by=account_id, - _environment_variables=[], - _conversation_variables=[], - ) - session.add(workflow) - session.commit() - return workflow - - monkeypatch.setattr(converter_module.WorkflowConverter, "convert_app_model_config_to_workflow", build_graph) - with sqlite_session_factory(expire_on_commit=True) as session: - account = make_account(account_id=str(uuid4()), interface_language="en-US") - source = make_app(app_id=source_id, mode=source_mode, tenant_id=str(uuid4())) - config = AppModelConfig(app_id=source_id) - source.app_model_config_id = config.id - session.add_all([account, source, config]) - session.commit() - - def request(method: str, endpoint: str, **kwargs: Any) -> dict[str, Any]: - assert not session.in_transaction(), "Enterprise I/O must run after conversion commits" - calls.append(method) - if method == "GET": - assert endpoint == "/webapp/access-mode/id" - assert kwargs["params"] == {"appId": source_id} - if outcome == "read_timeout": - raise httpx.ReadTimeout("source unavailable") - return {"accessMode": access_mode} - assert method == "POST" - assert endpoint == "/webapp/access-mode" - target_id = kwargs["json"]["appId"] - assert target_id != source_id - assert kwargs["json"]["accessMode"] == ("public" if outcome == "read_timeout" else access_mode) - if outcome == "write_error": - raise write_error - written[target_id] = kwargs["json"]["accessMode"] - return {"result": outcome != "false_result"} - - monkeypatch.setattr(EnterpriseRequest, "send_request", request) - service = WorkflowService(session_maker=sqlite_session_factory) - if outcome == "conversion_error": - with pytest.raises(ValueError, match="conversion failed"): - service.convert_to_workflow(source, account, {"name": "Converted"}, session=session) - assert calls == [] - return - if outcome == "write_error": - with pytest.raises(EnterpriseServiceError) as raised: - service.convert_to_workflow(source, account, {"name": "Converted"}, session=session) - assert raised.value is write_error - assert calls == ["GET", "POST"] - else: - target = service.convert_to_workflow(source, account, {"name": "Converted"}, session=session) - assert target.id != source_id - assert target.mode == (AppMode.ADVANCED_CHAT if source_mode == AppMode.CHAT else AppMode.WORKFLOW) - assert target.workflow_id is None - assert calls == ([] if outcome == "disabled" else ["GET", "POST"]) - if outcome != "disabled": - assert written == {target.id: "public" if outcome == "read_timeout" else access_mode} - - with sqlite_session_factory() as verification: - target = verification.scalar(select(App).where(App.name == "Converted")) - assert target is not None - assert verification.get(App, source_id) is not None - workflow = verification.scalar(select(Workflow).where(Workflow.app_id == target.id)) - assert workflow is not None - assert workflow.version == Workflow.VERSION_DRAFT - assert verification.scalar(select(Site).where(Site.app_id == target.id)) is not None - assert verification.scalar(select(InstalledApp).where(InstalledApp.app_id == target.id)) is not None + result = service.run_free_workflow_node({}, "tenant-1", "user-1", "node-1", {}) + assert result == node_execution + + def test_validate_graph_structure_coexist_error(self, service: WorkflowService) -> None: + graph = { + "nodes": [ + {"data": {"type": "start"}}, + {"data": {"type": "trigger-webhook"}}, # is_trigger_node=True + ] + } + with pytest.raises(ValueError, match="Start node and trigger nodes cannot coexist"): + service.validate_graph_structure(graph) + + def test_validate_features_structure_success(self, service: WorkflowService) -> None: + app = TestWorkflowAssociatedDataFactory.create_app(mode=AppMode.WORKFLOW) + features = {} + with patch("services.workflow_service.WorkflowAppConfigManager.config_validate") as mock_val: + service.validate_features_structure(app, features) + mock_val.assert_called_once() + + def test_validate_features_structure_invalid_mode(self, service: WorkflowService) -> None: + app = TestWorkflowAssociatedDataFactory.create_app() + app.mode = "invalid" + with pytest.raises(ValueError, match="Invalid app mode"): + service.validate_features_structure(app, {}) + + def test_validate_human_input_node_data_error(self, service: WorkflowService) -> None: + with patch("services.workflow_service.HumanInputNodeData.model_validate", side_effect=Exception("error")): + with pytest.raises(ValueError, match="Invalid HumanInput node data"): + service._validate_human_input_node_data({}) + + def test_rebuild_single_file_unreachable(self) -> None: + # Test line 1523 (unreachable) + with pytest.raises(Exception, match="unreachable"): + _rebuild_single_file("tenant-1", {}, cast(Any, "invalid_type")) diff --git a/api/tests/unit_tests/services/test_workflow_tool_source_service.py b/api/tests/unit_tests/services/test_workflow_tool_source_service.py index 918a5a87bf69af..e6edb1770b5395 100644 --- a/api/tests/unit_tests/services/test_workflow_tool_source_service.py +++ b/api/tests/unit_tests/services/test_workflow_tool_source_service.py @@ -22,7 +22,7 @@ def test_nested_tools_reject_v2_human_input_without_disabling_top_level_v2(versi workflow_kind=WorkflowKind.STANDARD, ) repository.get_by_provider_ids.return_value = [source] - service = WorkflowToolSourceService(repository) + service = WorkflowToolSourceService(repository, workflow_runtime=None, draft_variable_saver=None) # A top-level V2 node remains available in the same execution context. assert service.load(tenant_id="tenant", graph_config=form_graph).sources == {} tool_graph = {"nodes": [{"data": {"type": "tool", "provider_type": "workflow", "provider_id": "provider"}}]} diff --git a/api/tests/unit_tests/services/tools/test_api_provider_runtime.py b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py new file mode 100644 index 00000000000000..09328eb39732a0 --- /dev/null +++ b/api/tests/unit_tests/services/tools/test_api_provider_runtime.py @@ -0,0 +1,168 @@ +"""Exercise Agent API-tool invocation through real provider and credentials reads.""" + +import json +from collections.abc import Iterator +from typing import cast + +import httpx +import pytest +from sqlalchemy import Table, create_engine, event +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import QueuePool + +from core.db import session_factory +from core.tools.custom_tool.provider import ApiToolProviderController +from core.tools.entities.tool_bundle import ApiToolBundle +from core.tools.entities.tool_entities import ApiProviderSchemaType +from core.tools.errors import ToolInvokeError +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.ext_database import db +from models.account import Account +from models.base import TypeBase +from models.model import App +from models.tools import ApiToolProvider +from services.entities.agent_tool_inner import AgentToolInvokeRequest +from services.errors.agent_tool_inner import AgentToolInnerServiceError +from tests.helpers.unit.model_factories import make_account +from tests.unit_tests.services.agent.test_tool_invocation_service import ( + TENANT_ID, + USER_ID, + _persist_app, + _request, + _service, +) + +type Databases = tuple[sessionmaker[Session], Session] + + +@pytest.fixture +def databases(monkeypatch: pytest.MonkeyPatch) -> Iterator[Databases]: + engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + tables: list[Table] = [] + for model in (App, Account, ApiToolProvider): + table = model.__table__ + assert isinstance(table, Table) + tables.append(table) + TypeBase.metadata.create_all(engine, tables=tables) + sessions, global_sessions = [sessionmaker(engine, expire_on_commit=False) for engine in engines] + with global_sessions() as global_session: + monkeypatch.setattr(db, "session", global_session) + monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) + monkeypatch.setattr(session_factory, "create_session", global_sessions) + + def reject_global(*_args: object) -> None: + pytest.fail("Tool runtime queried the global database") + + event.listen(engines[1], "before_cursor_execute", reject_global) + try: + yield sessions, global_session + finally: + global_session.close() + for engine in engines: + engine.dispose() + + +@pytest.mark.parametrize("http_failure", [False, True]) +def test_api_tool_provider_read_finishes_before_decryption_and_http( + databases: Databases, monkeypatch: pytest.MonkeyPatch, http_failure: bool +) -> None: + sessions, global_session = databases + with sessions() as session: + _persist_app(session) + session.add(make_account(account_id=USER_ID, name="Injected author")) + provider = ApiToolProvider( + name="search", + icon="icon.svg", + schema="{}", + schema_type_str=ApiProviderSchemaType.OPENAPI, + tenant_id=TENANT_ID, + user_id=USER_ID, + description="Search", + credentials_str=json.dumps({"auth_type": "api_key_header", "api_key_value": "encrypted"}), + tools_str=json.dumps( + [ + ApiToolBundle( + server_url="https://api.example.com/search", + method="GET", + operation_id="search", + summary="Search", + author="Injected author", + parameters=[], + openapi={"parameters": []}, + ).model_dump(mode="json") + ] + ), + ) + session.add(provider) + session.commit() + engine = sessions.kw["bind"] + calls: list[str] = [] + + def released() -> None: + assert engine.pool.checkedout() == 0 + assert not global_session.in_transaction() + + class Decrypter: + def decrypt(self, credentials: dict[str, object]) -> dict[str, object]: + released() + assert credentials["api_key_value"] == "encrypted" + calls.append("decrypt") + return {**credentials, "api_key_value": "clear-secret"} + + def encryption(*, tenant_id: str, controller: ApiToolProviderController) -> tuple[Decrypter, None]: + released() + assert tenant_id == TENANT_ID + assert controller.entity.identity.author == "Injected author" + return Decrypter(), None + + def http_get(url: str, **kwargs: object) -> httpx.Response: + released() + assert url == "https://api.example.com/search" + headers = cast(dict[str, str], kwargs["headers"]) + assert headers["Authorization"] == "clear-secret" + calls.append("http") + if http_failure: + raise ToolInvokeError("remote unavailable") + return httpx.Response(200, text="ok", request=httpx.Request("GET", url)) + + monkeypatch.setattr("services.tools.tool_manager.create_tool_provider_encrypter", encryption) + monkeypatch.setattr("core.tools.custom_tool.tool.ssrf_proxy.get", http_get) + payload = _request().model_dump() + payload["tool"].update(provider_type="api", provider_id=provider.id, tool_parameters={}, runtime_parameters={}) + request = AgentToolInvokeRequest.model_validate(payload) + runtime = build_workflow_execution_dependencies(sessions) + with sessions() as session: + service = _service(session, runtime) + if http_failure: + with pytest.raises(AgentToolInnerServiceError) as exc: + service.invoke(request) + assert exc.value.error_code == "agent_tool_invoke_failed" + else: + assert service.invoke(request).observation == "ok" + assert calls == ["decrypt", "http"] + released() + + +def test_api_provider_lookup_is_tenant_scoped(databases: Databases) -> None: + sessions, _ = databases + with sessions.begin() as session: + provider = ApiToolProvider( + name="private", + icon="", + schema="{}", + schema_type_str=ApiProviderSchemaType.OPENAPI, + tenant_id=TENANT_ID, + user_id=USER_ID, + description="", + credentials_str="{}", + tools_str="[]", + ) + session.add(provider) + runtime = build_workflow_execution_dependencies(sessions) + assert runtime.tool_providers.get(tenant_id="other-tenant", provider_id=provider.id) is None + record = runtime.tool_providers.get(tenant_id=TENANT_ID, provider_id=provider.id) + assert record is not None + assert record.author == "" + assert record.tools == [] + assert sessions.kw["bind"].pool.checkedout() == 0 diff --git a/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py new file mode 100644 index 00000000000000..2437468f8515aa --- /dev/null +++ b/api/tests/unit_tests/services/tools/test_builtin_provider_runtime.py @@ -0,0 +1,330 @@ +"""Exercise selected builtin credentials through real repositories and real tool construction.""" + +import json +from collections.abc import Generator, Iterator +from typing import cast +from unittest.mock import create_autospec +from uuid import uuid4 + +import pytest +from sqlalchemy import Engine, Table, create_engine, event +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import QueuePool + +from core.db import session_factory +from core.plugin.entities.plugin_daemon import CredentialType, PluginOAuthCredentialsResponse +from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin, ToolProviderType +from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError +from core.tools.plugin_tool.provider import PluginToolProviderController +from extensions.application_services import tools as tool_services_module +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.ext_database import db +from models.account import Account +from models.base import TypeBase +from models.tools import BuiltinToolProvider, MCPToolProvider, ToolOAuthSystemClient, ToolOAuthTenantClient +from services.credentials import tool_runtime_adapters as credential_runtime_module +from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime +from services.entities.agent_tool_inner import AgentToolInvokeRequest +from services.errors.agent_tool_inner import AgentToolInnerServiceError +from services.tools.agent_invocation_gateway import AgentToolInvocationGateway +from services.tools.builtin import credentials as credentials_module +from services.tools.tool_manager import ToolManager +from services.workflow.variable_contracts import WorkflowExecutionVariables +from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLock + +TENANT = "tenant" +PROVIDER = "vendor/search/search" + + +@pytest.fixture +def databases( + monkeypatch: pytest.MonkeyPatch, +) -> Generator[tuple[sessionmaker[Session], Session]]: + engines: list[Engine] = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + TypeBase.metadata.create_all( + engine, + tables=cast( + list[Table], + [ + model.__table__ + for model in ( + Account, + BuiltinToolProvider, + ToolOAuthTenantClient, + ToolOAuthSystemClient, + MCPToolProvider, + ) + ], + ), + ) + sessions: sessionmaker[Session] = sessionmaker(engines[0], expire_on_commit=False) + global_sessions: sessionmaker[Session] = sessionmaker(engines[1], expire_on_commit=False) + with global_sessions() as global_session: + monkeypatch.setattr(db, "session", global_session) + monkeypatch.setattr(type(db), "engine", property(lambda _db: engines[1])) + monkeypatch.setattr(session_factory, "create_session", global_sessions) + + def reject_global(*_args: object) -> None: + pytest.fail("Tool runtime read the global database") + + event.listen(engines[1], "before_cursor_execute", reject_global) + try: + yield sessions, global_session + finally: + global_session.close() + for engine in engines: + engine.dispose() + + +def seed( + sessions: sessionmaker[Session], + *, + expires_at: int = 1, + oauth_client: str = "tenant", +) -> str: + with sessions.begin() as session: + credential = BuiltinToolProvider( + tenant_id=TENANT, + user_id="owner", + provider=PROVIDER, + name="Selected", + encrypted_credentials=json.dumps({"token": "encrypted-old"}), + credential_type=CredentialType.OAUTH2, + expires_at=expires_at, + ) + session.add(credential) + if oauth_client == "tenant": + client = ToolOAuthTenantClient(tenant_id=TENANT, plugin_id="vendor/search", provider="search") + client.encrypted_oauth_params = json.dumps({"client": "encrypted-client"}) + session.add(client) + elif oauth_client == "system": + session.add( + ToolOAuthSystemClient(plugin_id="vendor/search", provider="search", encrypted_oauth_params="system") + ) + return credential.id + + +def controller() -> PluginToolProviderController: + entity = ToolProviderEntityWithPlugin.model_validate( + { + "identity": { + "author": "author", + "name": PROVIDER, + "description": {"en_US": "Search"}, + "icon": "icon.svg", + "label": {"en_US": "Search"}, + }, + "credentials_schema": [ + {"name": "token", "label": {"en_US": "Token"}, "type": "secret-input", "required": True} + ], + "oauth_schema": {"client_schema": [], "credentials_schema": []}, + "tools": [ + {"identity": {"author": "author", "name": "search", "provider": PROVIDER, "label": {"en_US": "Search"}}} + ], + } + ) + return PluginToolProviderController( + entity=entity, plugin_id="vendor/search", plugin_unique_identifier="vendor/search:1", tenant_id=TENANT + ) + + +@pytest.mark.parametrize("client", ["tenant", "system"]) +@pytest.mark.parametrize("outcome", ["success", "refresh-error", "concurrent-edit", "concurrent-delete"]) +def test_selected_oauth_credential_releases_connections_across_full_invocation( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + client: str, + outcome: str, +) -> None: + sessions, global_session = databases + credential_id = seed(sessions, oauth_client=client) + runtime = build_workflow_execution_dependencies(sessions) + refresh_lock = ThreadCredentialRefreshLock() + calls: list[str] = [] + + def released() -> None: + assert sessions.kw["bind"].pool.checkedout() == 0 + assert not global_session.in_transaction() + + class Codec: + def decrypt(self, values: dict[str, str]) -> dict[str, str]: + released() + calls.append("decrypt") + return {k: v.removeprefix("encrypted-") for k, v in values.items()} + + def encrypt(self, values: dict[str, str]) -> dict[str, str]: + released() + calls.append("encrypt") + return {k: "encrypted-" + v for k, v in values.items()} + + class Cache: + def delete(self) -> None: + released() + calls.append("invalidate") + + def encrypter(**_kwargs: object) -> tuple[Codec, Cache]: + released() + return Codec(), Cache() + + def policy(**kwargs: object) -> None: + released() + assert kwargs["check_existence"] is False + assert kwargs["credential_id"] == credential_id + calls.append("policy") + + def refresh(_self: object, **kwargs: object) -> PluginOAuthCredentialsResponse: + released() + calls.append("refresh") + assert kwargs["credentials"] == {"token": "old"} + assert kwargs["system_credentials"] == {"client": "client"} + if outcome == "refresh-error": + raise ValueError("broker unavailable") + if outcome in {"concurrent-edit", "concurrent-delete"}: + with sessions.begin() as session: + record = session.get(BuiltinToolProvider, credential_id) + assert record is not None + if outcome == "concurrent-delete": + session.delete(record) + else: + record.encrypted_credentials = '{"token":"manual-edit"}' + return PluginOAuthCredentialsResponse(credentials={"token": "new"}, expires_at=4_000_000_000) + + def invoke(_self: object, **kwargs: object) -> Iterator[object]: + released() + calls.append("invoke") + assert kwargs["credentials"] == {"token": "new"} + return iter(()) + + def verified(*_args: object) -> bool: + released() + calls.append("verified") + return True + + def decrypt_system(params: str) -> dict[str, str]: + released() + assert params == "system" + return {"client": "client"} + + monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + monkeypatch.setattr(ToolManager, "_credential_resolver", tool_services_module.resolve_tool_credentials) + monkeypatch.setattr(tool_services_module, "get_session_maker", lambda: sessions) + monkeypatch.setattr(tool_services_module, "RedisCredentialRefreshLock", lambda **_kwargs: refresh_lock) + monkeypatch.setattr(credentials_module, "create_provider_encrypter", encrypter) + monkeypatch.setattr(credential_runtime_module, "create_provider_encrypter", encrypter) + monkeypatch.setattr(credential_runtime_module, "runtime_check_credential_policy_compliance", policy) + monkeypatch.setattr(credential_runtime_module.OAuthHandler, "refresh_credentials", refresh) + monkeypatch.setattr(ProviderToolCredentialRuntime, "_cache", lambda _self, _record: Cache()) + monkeypatch.setattr(credentials_module.PluginService, "is_plugin_verified", verified) + monkeypatch.setattr(credentials_module, "decrypt_system_params", decrypt_system) + monkeypatch.setattr("core.plugin.impl.tool.PluginToolManager.invoke", invoke) + + variables = create_autospec(WorkflowExecutionVariables, instance=True, spec_set=True) + variables.saver_factory.side_effect = AssertionError("No workflow variables for this tool") + + request = AgentToolInvokeRequest.model_validate( + { + "caller": { + "tenant_id": TENANT, + "user_id": "caller", + "user_from": "account", + "app_id": "app", + "invoke_from": "debugger", + }, + "tool": { + "provider_type": "builtin", + "provider_id": PROVIDER, + "tool_name": "search", + "credential_id": credential_id, + }, + } + ) + gateway = AgentToolInvocationGateway(variables=variables, runtime=runtime) + if outcome == "success": + gateway.invoke(request) + assert calls[-1] == "invoke" + else: + with pytest.raises(AgentToolInnerServiceError) as exc: + gateway.invoke(request) + assert exc.value.error_code == "agent_tool_credential_invalid" + assert "invoke" not in calls + assert refresh_lock.keys == [("tool", TENANT, credential_id)] + released() + with sessions() as session: + persisted = session.get(BuiltinToolProvider, credential_id) + if outcome == "concurrent-delete": + assert persisted is None + else: + assert persisted is not None + expected = {"success": "encrypted-new", "refresh-error": "encrypted-old", "concurrent-edit": "manual-edit"}[ + outcome + ] + assert persisted.credentials == {"token": expected} + assert persisted.expires_at == (4_000_000_000 if outcome == "success" else 1) + + +@pytest.mark.parametrize("wrong_scope", ["tenant", "provider"]) +def test_selected_credential_cannot_cross_owner_scope( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + wrong_scope: str, +) -> None: + sessions, _ = databases + credential_id = seed(sessions, expires_at=-1) + with sessions.begin() as session: + record = session.get(BuiltinToolProvider, credential_id) + assert record is not None + if wrong_scope == "tenant": + record.tenant_id = "other" + else: + record.provider = "other/provider" + monkeypatch.setattr(ToolManager, "get_builtin_provider", lambda *_args: controller()) + monkeypatch.setattr(ToolManager, "_credential_resolver", tool_services_module.resolve_tool_credentials) + monkeypatch.setattr(tool_services_module, "get_session_maker", lambda: sessions) + runtime = build_workflow_execution_dependencies(sessions) + with pytest.raises(ToolProviderCredentialValidationError, match="has been deleted"): + ToolManager.get_tool_runtime( + provider_type=ToolProviderType.BUILT_IN, + provider_id=PROVIDER, + tool_name="search", + tenant_id=TENANT, + credential_id=credential_id, + tool_providers=runtime.tool_providers, + workflow_queries=runtime.tools, + ) + + +@pytest.mark.parametrize("reference", ["id", "identifier", "uuid-identifier", "missing", "other-tenant"]) +def test_mcp_runtime_resolves_injected_provider_before_decryption( + databases: tuple[sessionmaker[Session], Session], + monkeypatch: pytest.MonkeyPatch, + reference: str, +) -> None: + sessions, _ = databases + with sessions.begin() as session: + record = MCPToolProvider( + name="server", + server_identifier=str(uuid4()) if reference == "uuid-identifier" else "server", + server_url="encrypted-url", + server_url_hash="hash", + icon="icon.svg", + tenant_id=TENANT, + user_id="owner", + ) + session.add(record) + + def build(provider: MCPToolProvider) -> str: + assert sessions.kw["bind"].pool.checkedout() == 0 + assert provider.id == record.id + return "controller" + + monkeypatch.setattr("services.tools.tool_manager.MCPToolProviderController.from_db", build) + key = record.id if reference == "id" else "missing" if reference == "missing" else record.server_identifier + providers = build_workflow_execution_dependencies(sessions).tool_providers + if reference in {"missing", "other-tenant"}: + with pytest.raises(ToolProviderNotFoundError): + ToolManager.get_mcp_provider_controller( + "other" if reference == "other-tenant" else TENANT, key, tool_providers=providers + ) + else: + assert ToolManager.get_mcp_provider_controller(TENANT, key, tool_providers=providers) == "controller" diff --git a/api/tests/unit_tests/services/tracing/test_recovery.py b/api/tests/unit_tests/services/tracing/test_recovery.py index b0a4ffc06d4ec2..e183f90a0be884 100644 --- a/api/tests/unit_tests/services/tracing/test_recovery.py +++ b/api/tests/unit_tests/services/tracing/test_recovery.py @@ -125,9 +125,9 @@ def save_agent_stop( ) from core.app.entities.app_invoke_entities import InvokeFrom from core.app.entities.queue_entities import QueueStopEvent - from core.app.task_pipeline.easy_ui_based_generate_task_pipeline import EasyUIBasedGenerateTaskPipeline from core.ops import ops_trace_manager from core.ops.base_trace_instance import bind_application_tracing_hooks + from services.app.generation.adapters.message_pipeline import EasyUIBasedGenerateTaskPipeline from services.tracing import runtime_bridge from tests.helpers.unit.config_override import apply_config_overrides from tests.helpers.unit.core.model_fixtures import make_model_config @@ -145,9 +145,9 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity if notify is not None: notify(message, application_generate_entity) - monkeypatch.setattr("core.app.task_pipeline.easy_ui_based_generate_task_pipeline.message_was_created.send", send) + monkeypatch.setattr("services.app.generation.adapters.message_pipeline.message_was_created.send", send) monkeypatch.setattr( - "core.app.task_pipeline.easy_ui_based_generate_task_pipeline.naive_utc_now", + "repositories.app.generation_repository.naive_utc_now", lambda: case.now - timedelta(seconds=10), ) with case.collection.sessions.begin() as session: @@ -179,7 +179,14 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity invoke_from=InvokeFrom.DEBUGGER, ) queue_manager = _QueueManager() - pipeline = EasyUIBasedGenerateTaskPipeline(entity, queue_manager, conversation, message, stream=True) + pipeline = EasyUIBasedGenerateTaskPipeline( + entity, + queue_manager, + conversation, + message, + stream=True, + records=AppGenerationRepository(case.collection.sessions), + ) pipeline._task_state.llm_result.message.content = answer application = Flask(__name__) bind_application_tracing_hooks(application, runtime_bridge.RuntimeApplicationTracingHooks()) @@ -192,11 +199,9 @@ def send(message: Message, *, application_generate_entity: ChatAppGenerateEntity monkeypatch.setattr(pipeline, "_handle_stop", lambda _event: None) monkeypatch.setattr(pipeline, "handle_output_moderation_when_task_finished", lambda _answer: None) list(pipeline._process_stream_response(publisher=None, trace_manager=trace_manager)) - elif consumer_session is None: - with case.collection.sessions.begin() as session: - pipeline._save_message(session=session, trace_manager=trace_manager, preserve_existing_usage=True) else: - pipeline._save_message(session=consumer_session, trace_manager=trace_manager, preserve_existing_usage=True) + pipeline._save_message(trace_manager=trace_manager, preserve_existing_usage=True) + if consumer_session is not None: consumer_session.commit() return trace_manager.collect_tasks() @@ -370,10 +375,13 @@ def test_classic_stopped_consumer_keeps_its_existing_trace_notification( def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( - case: RecoveryCase, monkeypatch: pytest.MonkeyPatch + case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, app_records: AppGenerationRepository ) -> None: - from core.app.apps.agent_app.app_runner import AgentAppRunner + from clients.agent_backend import FakeAgentBackendRunClient from graphon.model_runtime.entities.llm_entities import LLMUsage + from services.app.generation.adapters.agent_runner import AgentAppRunner + from services.app.generation.ports import MessageIdentity + from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner # The worker's scoped session may already have loaded this Message before # the consumer commits its independent stop transaction. @@ -381,8 +389,14 @@ def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( assert cached_message is not None assert cached_message.message_metadata is None save_agent_stop(case, monkeypatch, "partial answer") - AgentAppRunner._persist_message_usage( - message_id=cached_message.id, + runner: AgentAppRunner = _runner(FakeAgentBackendRunClient(), _FakeSessionStore(), app_records=app_records) + runner._persist_message_usage( + identity=MessageIdentity( + case.collection.run.tenant_id, + case.collection.run.app_id, + cached_message.conversation_id, + cached_message.id, + ), usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}), ) with case.collection.sessions() as session: @@ -396,10 +410,13 @@ def test_worker_usage_flush_preserves_stop_intent_from_a_newer_consumer_save( def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_read( - case: RecoveryCase, monkeypatch: pytest.MonkeyPatch + case: RecoveryCase, monkeypatch: pytest.MonkeyPatch, app_records: AppGenerationRepository ) -> None: - from core.app.apps.agent_app.app_runner import AgentAppRunner + from clients.agent_backend import FakeAgentBackendRunClient from graphon.model_runtime.entities.llm_entities import LLMUsage + from services.app.generation.adapters.agent_runner import AgentAppRunner + from services.app.generation.ports import MessageIdentity + from tests.unit_tests.core.app.apps.agent_app.test_app_runner import _FakeSessionStore, _runner with case.collection.sessions.begin() as session: initial = session.get(Message, case.collection.run.source_id) @@ -410,8 +427,14 @@ def test_stop_save_preserves_worker_usage_committed_after_its_cached_message_rea cached = consumer.get(Message, case.collection.run.source_id) assert cached is not None assert cached.message_tokens == 0 - AgentAppRunner._persist_message_usage( - message_id=cached.id, + runner: AgentAppRunner = _runner(FakeAgentBackendRunClient(), _FakeSessionStore(), app_records=app_records) + runner._persist_message_usage( + identity=MessageIdentity( + case.collection.run.tenant_id, + case.collection.run.app_id, + cached.conversation_id, + cached.id, + ), usage=LLMUsage.from_metadata({"prompt_tokens": 13, "completion_tokens": 8, "total_price": "0.21"}), ) save_agent_stop(case, monkeypatch, "partial answer", consumer_session=consumer) diff --git a/api/tests/unit_tests/services/trigger/test_schedule_service.py b/api/tests/unit_tests/services/trigger/test_schedule_service.py index a68123ed6e5c45..25f293b26f5b4b 100644 --- a/api/tests/unit_tests/services/trigger/test_schedule_service.py +++ b/api/tests/unit_tests/services/trigger/test_schedule_service.py @@ -2,7 +2,6 @@ import unittest from datetime import UTC, datetime from typing import Any -from unittest.mock import MagicMock import pytest @@ -11,7 +10,9 @@ from core.workflow.nodes.trigger_schedule.exc import ScheduleConfigError from libs.schedule_utils import calculate_next_run_at, convert_12h_to_24h from models.workflow import Workflow +from services.trigger.schedule_policy import visual_to_cron from services.trigger.schedule_service import ScheduleService +from services.trigger.workflow_policy import schedule_config from tests.helpers.unit.model_factories import make_workflow @@ -90,13 +91,13 @@ class TestVisualToCron(unittest.TestCase): def test_visual_to_cron_hourly(self): """Test converting hourly visual config to cron.""" visual_config = VisualConfig(on_minute=15) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "15 * * * *" def test_visual_to_cron_daily(self): """Test converting daily visual config to cron.""" visual_config = VisualConfig(time="2:30 PM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "30 14 * * *" def test_visual_to_cron_weekly(self): @@ -105,7 +106,7 @@ def test_visual_to_cron_weekly(self): time="10:00 AM", weekdays=["mon", "wed", "fri"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "0 10 * * 1,3,5" def test_visual_to_cron_monthly_with_specific_days(self): @@ -114,7 +115,7 @@ def test_visual_to_cron_monthly_with_specific_days(self): time="11:30 AM", monthly_days=[1, 15], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 11 1,15 * *" def test_visual_to_cron_monthly_with_last_day(self): @@ -123,7 +124,7 @@ def test_visual_to_cron_monthly_with_last_day(self): time="11:30 AM", monthly_days=[1, "last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 11 1,L * *" def test_visual_to_cron_monthly_only_last_day(self): @@ -132,7 +133,7 @@ def test_visual_to_cron_monthly_only_last_day(self): time="9:00 PM", monthly_days=["last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "0 21 L * *" def test_visual_to_cron_monthly_with_end_days_and_last(self): @@ -141,46 +142,46 @@ def test_visual_to_cron_monthly_with_end_days_and_last(self): time="3:45 PM", monthly_days=[29, 30, 31, "last"], ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) # Should have 29,30,31,L - the L handles all possible last days assert result == "45 15 29,30,31,L * *" def test_visual_to_cron_invalid_frequency(self): """Test converting with invalid frequency.""" with pytest.raises(ScheduleConfigError, match="Unsupported frequency: invalid"): - ScheduleService.visual_to_cron("invalid", VisualConfig()) + visual_to_cron("invalid", VisualConfig()) def test_visual_to_cron_weekly_no_weekdays(self): """Test converting weekly with no weekdays specified.""" visual_config = VisualConfig(time="10:00 AM") with pytest.raises(ScheduleConfigError, match="Weekdays are required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) def test_visual_to_cron_hourly_no_minute(self): """Test converting hourly with no on_minute specified.""" visual_config = VisualConfig() # on_minute defaults to 0 - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "0 * * * *" # Should use default value 0 def test_visual_to_cron_daily_no_time(self): """Test converting daily with no time specified.""" visual_config = VisualConfig(time=None) with pytest.raises(ScheduleConfigError, match="time is required for daily schedules"): - ScheduleService.visual_to_cron("daily", visual_config) + visual_to_cron("daily", visual_config) def test_visual_to_cron_weekly_no_time(self): """Test converting weekly with no time specified.""" visual_config = VisualConfig(weekdays=["mon"]) visual_config.time = None # Override default with pytest.raises(ScheduleConfigError, match="time is required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) def test_visual_to_cron_monthly_no_time(self): """Test converting monthly with no time specified.""" visual_config = VisualConfig(monthly_days=[1]) visual_config.time = None # Override default with pytest.raises(ScheduleConfigError, match="time is required for monthly schedules"): - ScheduleService.visual_to_cron("monthly", visual_config) + visual_to_cron("monthly", visual_config) def test_visual_to_cron_monthly_duplicate_days(self): """Test monthly with duplicate days should be deduplicated.""" @@ -188,7 +189,7 @@ def test_visual_to_cron_monthly_duplicate_days(self): time="10:00 AM", monthly_days=[1, 15, 1, 15, 31], # Duplicates ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "0 10 1,15,31 * *" # Should be deduplicated def test_visual_to_cron_monthly_unsorted_days(self): @@ -197,7 +198,7 @@ def test_visual_to_cron_monthly_unsorted_days(self): time="2:30 PM", monthly_days=[20, 5, 15, 1, 10], # Unsorted ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "30 14 1,5,10,15,20 * *" # Should be sorted def test_visual_to_cron_weekly_all_weekdays(self): @@ -206,31 +207,31 @@ def test_visual_to_cron_weekly_all_weekdays(self): time="8:00 AM", weekdays=["sun", "mon", "tue", "wed", "thu", "fri", "sat"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "0 8 * * 0,1,2,3,4,5,6" def test_visual_to_cron_hourly_boundary_values(self): """Test hourly with boundary minute values.""" # Minimum value visual_config = VisualConfig(on_minute=0) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "0 * * * *" # Maximum value visual_config = VisualConfig(on_minute=59) - result = ScheduleService.visual_to_cron("hourly", visual_config) + result = visual_to_cron("hourly", visual_config) assert result == "59 * * * *" def test_visual_to_cron_daily_midnight_noon(self): """Test daily at special times (midnight and noon).""" # Midnight visual_config = VisualConfig(time="12:00 AM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "0 0 * * *" # Noon visual_config = VisualConfig(time="12:00 PM") - result = ScheduleService.visual_to_cron("daily", visual_config) + result = visual_to_cron("daily", visual_config) assert result == "0 12 * * *" def test_visual_to_cron_monthly_mixed_with_last_and_duplicates(self): @@ -239,7 +240,7 @@ def test_visual_to_cron_monthly_mixed_with_last_and_duplicates(self): time="11:45 PM", monthly_days=[15, 1, "last", 15, 30, 1, "last"], # Mixed with duplicates ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) assert result == "45 23 1,15,30,L * *" # Deduplicated and sorted with L at end def test_visual_to_cron_weekly_single_day(self): @@ -248,7 +249,7 @@ def test_visual_to_cron_weekly_single_day(self): time="6:30 PM", weekdays=["sun"], ) - result = ScheduleService.visual_to_cron("weekly", visual_config) + result = visual_to_cron("weekly", visual_config) assert result == "30 18 * * 0" def test_visual_to_cron_monthly_all_possible_days(self): @@ -258,7 +259,7 @@ def test_visual_to_cron_monthly_all_possible_days(self): time="12:01 AM", monthly_days=all_days, ) - result = ScheduleService.visual_to_cron("monthly", visual_config) + result = visual_to_cron("monthly", visual_config) expected_days = ",".join([str(i) for i in range(1, 32)]) + ",L" assert result == f"1 0 {expected_days} * *" @@ -266,13 +267,13 @@ def test_visual_to_cron_monthly_no_days(self): """Test monthly without any days specified should raise error.""" visual_config = VisualConfig(time="10:00 AM", monthly_days=[]) with pytest.raises(ScheduleConfigError, match="Monthly days are required for monthly schedules"): - ScheduleService.visual_to_cron("monthly", visual_config) + visual_to_cron("monthly", visual_config) def test_visual_to_cron_weekly_empty_weekdays_list(self): """Test weekly with empty weekdays list should raise error.""" visual_config = VisualConfig(time="10:00 AM", weekdays=[]) with pytest.raises(ScheduleConfigError, match="Weekdays are required for weekly schedules"): - ScheduleService.visual_to_cron("weekly", visual_config) + visual_to_cron("weekly", visual_config) class TestParseTime(unittest.TestCase): @@ -351,7 +352,7 @@ def test_extract_schedule_config_with_cron_mode(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is not None assert config.node_id == "schedule-node" @@ -379,7 +380,7 @@ def test_extract_schedule_config_with_visual_mode(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is not None assert config.node_id == "schedule-node" @@ -401,7 +402,7 @@ def test_extract_schedule_config_no_schedule_node(self): ), ) - config = ScheduleService.extract_schedule_config(workflow) + config = schedule_config(workflow.graph_dict) assert config is None def test_extract_schedule_config_invalid_graph(self): @@ -411,7 +412,7 @@ def test_extract_schedule_config_invalid_graph(self): ) with pytest.raises(ScheduleConfigError, match="Workflow graph is empty"): - ScheduleService.extract_schedule_config(workflow) + schedule_config(workflow.graph_dict) class TestScheduleWithTimezone(unittest.TestCase): @@ -430,7 +431,7 @@ def test_visual_schedule_with_timezone_integration(self): ) # Convert to cron expression - cron_expr = ScheduleService.visual_to_cron("monthly", visual_config) + cron_expr = visual_to_cron("monthly", visual_config) assert cron_expr is not None assert cron_expr == "30 10 1 * *" # Direct conversion @@ -463,7 +464,7 @@ def test_visual_schedule_different_timezones_same_local_time(self): weekdays=["mon"], ) - cron_expr = ScheduleService.visual_to_cron("weekly", visual_config) + cron_expr = visual_to_cron("weekly", visual_config) assert cron_expr is not None assert cron_expr == "0 9 * * 1" @@ -494,7 +495,7 @@ def test_visual_schedule_daily_across_dst_change(self): time="10:00 AM", ) - cron_expr = ScheduleService.visual_to_cron("daily", visual_config) + cron_expr = visual_to_cron("daily", visual_config) assert cron_expr is not None assert cron_expr == "0 10 * * *" @@ -547,7 +548,7 @@ def test_to_schedule_config_should_raise_for_cron_mode_without_expression() -> N ScheduleService.to_schedule_config(node_config=node_config) -def test_to_schedule_config_should_build_from_visual_mode(monkeypatch: pytest.MonkeyPatch) -> None: +def test_to_schedule_config_should_build_from_visual_mode() -> None: # Arrange node_config = { "id": "node-1", @@ -558,7 +559,6 @@ def test_to_schedule_config_should_build_from_visual_mode(monkeypatch: pytest.Mo "timezone": "UTC", }, } - monkeypatch.setattr(ScheduleService, "visual_to_cron", MagicMock(return_value="30 9 * * *")) # Act result = ScheduleService.to_schedule_config(node_config=node_config) @@ -582,7 +582,7 @@ def test_extract_schedule_config_should_raise_when_graph_is_empty() -> None: # Act / Assert with pytest.raises(ScheduleConfigError, match="Workflow graph is empty"): - ScheduleService.extract_schedule_config(workflow=workflow) + schedule_config(workflow.graph_dict) def test_extract_schedule_config_should_raise_when_mode_invalid() -> None: @@ -603,7 +603,7 @@ def test_extract_schedule_config_should_raise_when_mode_invalid() -> None: # Act / Assert with pytest.raises(ScheduleConfigError, match="Invalid schedule mode: invalid"): - ScheduleService.extract_schedule_config(workflow=workflow) + schedule_config(workflow.graph_dict) if __name__ == "__main__": diff --git a/api/tests/unit_tests/services/trigger/test_webhook_service.py b/api/tests/unit_tests/services/trigger/test_webhook_service.py index fe1a369277ffe5..fec89732605f28 100644 --- a/api/tests/unit_tests/services/trigger/test_webhook_service.py +++ b/api/tests/unit_tests/services/trigger/test_webhook_service.py @@ -1,19 +1,16 @@ import logging from io import BytesIO from types import SimpleNamespace -from typing import Any, cast +from typing import Any from unittest.mock import MagicMock, patch import pytest from flask import Flask -from sqlalchemy import Engine as _webhook_service_relationships_Engine from sqlalchemy import select from sqlalchemy.engine import Engine as _webhook_service_Engine from sqlalchemy.orm import Session, sessionmaker from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import RequestEntityTooLarge -from core.trigger.constants import TRIGGER_WEBHOOK_NODE_TYPE from core.workflow.nodes.trigger_webhook.entities import ( ContentType, WebhookBodyParameter, @@ -26,10 +23,11 @@ from models.tools import ToolFile from models.trigger import AppTrigger, WorkflowWebhookTrigger from models.workflow import Workflow +from repositories.trigger.workflow_repository import WorkflowTriggerRepository from services.errors.app import QuotaExceededError -from services.trigger import webhook_service from services.trigger import webhook_service as service_module from services.trigger import webhook_service as webhook_service_module +from services.trigger.errors import WebhookBodyTooLargeError from services.trigger.webhook_service import WebhookService from tests.helpers.unit.model_factories import make_app, make_end_user, make_workflow @@ -1028,7 +1026,7 @@ def test_extract_webhook_data_should_raise_for_request_too_large( apply_config_overrides(monkeypatch, WEBHOOK_REQUEST_BODY_MAX_SIZE=1) with flask_app.test_request_context("/webhook", method="POST", data="ab"): - with pytest.raises(RequestEntityTooLarge): + with pytest.raises(WebhookBodyTooLargeError): WebhookService.extract_webhook_data(_workflow_trigger()) def test_extract_octet_stream_body_should_return_none_when_empty_payload(self, flask_app: Flask) -> None: @@ -1257,7 +1255,7 @@ def test_generate_webhook_response_should_fallback_when_response_body_is_not_jso assert "message" in body def test_generate_webhook_id_should_return_24_character_identifier(self) -> None: - webhook_id = WebhookService.generate_webhook_id() + webhook_id = WorkflowTriggerRepository.generate_webhook_id() assert isinstance(webhook_id, str) assert len(webhook_id) == 24 @@ -1270,81 +1268,65 @@ def test_sanitize_key_should_return_original_value_for_non_string_input(self) -> class TestWebhookServiceRelationshipsCases: @staticmethod def test_draft_sync_preserves_webhook_id_when_deleted_node_is_restored( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: _webhook_service_relationships_Engine + sqlite_session_factory: sessionmaker[Session], ) -> None: - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), - ) + app = make_app(created_by="account-1") assert app.created_by is not None - webhook_trigger = WorkflowWebhookTrigger( + trigger = WorkflowWebhookTrigger( app_id=app.id, tenant_id=app.tenant_id, node_id="webhook-node", webhook_id="stable-webhook-id", created_by=app.created_by, ) - with session_maker.begin() as session: - session.add(webhook_trigger) - original_record_id = webhook_trigger.id - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) - - deleted_draft = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, deleted_draft, remove_stale=False) - - restored_draft = cast( - Workflow, - SimpleNamespace( - walk_nodes=lambda _node_type: iter([("webhook-node", {"type": TRIGGER_WEBHOOK_NODE_TYPE})]) - ), + with sqlite_session_factory.begin() as session: + session.add_all([app, trigger]) + original_id = trigger.id + WebhookService.sync_webhook_relationships( + app, + make_workflow(), + remove_stale=False, + session_factory=sqlite_session_factory, ) - WebhookService.sync_webhook_relationships(app, restored_draft, remove_stale=False) - - with Session(sqlite_engine) as session: + restored = make_workflow( + graph={"nodes": [{"id": "webhook-node", "data": {"type": "trigger-webhook"}}], "edges": []} + ) + WebhookService.sync_webhook_relationships( + app, + restored, + remove_stale=False, + session_factory=sqlite_session_factory, + ) + with sqlite_session_factory() as session: records = session.scalars(select(WorkflowWebhookTrigger)).all() - - assert len(records) == 1 - assert records[0].id == original_record_id - assert records[0].webhook_id == "stable-webhook-id" + assert len(records) == 1 + assert records[0].id == original_id + assert records[0].webhook_id == "stable-webhook-id" @staticmethod def test_published_sync_removes_stale_webhook_relationships( - monkeypatch: pytest.MonkeyPatch, sqlite_engine: _webhook_service_relationships_Engine + sqlite_session_factory: sessionmaker[Session], ) -> None: - session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False) - app = cast( - App, - SimpleNamespace(id="00000000-0000-0000-0000-000000000001", tenant_id="tenant-1", created_by="user-1"), - ) + app = make_app(created_by="account-1") assert app.created_by is not None - webhook_trigger = WorkflowWebhookTrigger( - app_id=app.id, - tenant_id=app.tenant_id, - node_id="deleted-webhook-node", - webhook_id="deleted-webhook-id", - created_by=app.created_by, - ) - with session_maker.begin() as session: - session.add(webhook_trigger) - - redis = MagicMock() - redis.get.return_value = None - redis.lock.return_value.acquire.return_value = True - monkeypatch.setattr(webhook_service, "db", SimpleNamespace(engine=sqlite_engine)) - monkeypatch.setattr(webhook_service, "redis_client", redis) - - published_workflow = cast(Workflow, SimpleNamespace(walk_nodes=lambda _node_type: iter(()))) - WebhookService.sync_webhook_relationships(app, published_workflow, remove_stale=True) - - with session_maker() as session: - records = session.scalars(select(WorkflowWebhookTrigger)).all() - assert records == [] - redis.delete.assert_called_once_with( - f"{WebhookService.__WEBHOOK_NODE_CACHE_KEY__}:{app.id}:deleted-webhook-node" + with sqlite_session_factory.begin() as session: + session.add_all( + [ + app, + WorkflowWebhookTrigger( + app_id=app.id, + tenant_id=app.tenant_id, + node_id="deleted-webhook-node", + webhook_id="deleted-webhook-id", + created_by=app.created_by, + ), + ] + ) + WebhookService.sync_webhook_relationships( + app, + make_workflow(), + remove_stale=True, + session_factory=sqlite_session_factory, ) + with sqlite_session_factory() as session: + assert session.scalars(select(WorkflowWebhookTrigger)).all() == [] diff --git a/api/tests/unit_tests/services/workflow/test_conversion_service.py b/api/tests/unit_tests/services/workflow/test_conversion_service.py new file mode 100644 index 00000000000000..168e335070d155 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_conversion_service.py @@ -0,0 +1,276 @@ +"""Real conversion, isolated persistence, and failures at the atomic write boundary.""" + +import json +from collections.abc import Mapping +from typing import cast + +import pytest +from sqlalchemy import event, func, select +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, SessionTransaction, sessionmaker + +from machinery.context import RequestContext +from models.account import TenantAccountJoin, TenantAccountRole +from models.api_based_extension import APIBasedExtension +from models.model import App, AppMode, AppModelConfig, InstalledApp, Site +from models.workflow import Workflow +from repositories.app.console_repository import ConsoleAppRepository +from services.app.console_service import ConsoleAppNotFoundError +from services.entities.app_entities import AppEvent +from services.errors.workflow_service import WorkflowConversionError +from services.workflow.conversion_service import WorkflowConversionService +from services.workflow.workflow_converter import WorkflowConverter +from tests.helpers.unit.model_factories import make_account, make_app, make_tenant + +CONTEXT = RequestContext("conversion", None, "account-1", "tenant-1") + + +@pytest.fixture +def sessions(sqlite_engine: Engine) -> sessionmaker[Session]: + class ConversionSession(Session): + pass + + factory = sessionmaker(sqlite_engine, class_=ConversionSession, expire_on_commit=False) + extension = APIBasedExtension( + tenant_id="tenant-1", name="Weather", api_endpoint="https://example.com/weather", api_key="encrypted" + ) + extension.id = "extension-1" + with factory.begin() as session: + session.add_all( + [ + make_account(), + make_tenant(), + TenantAccountJoin( + tenant_id=CONTEXT.active_workspace_id, + account_id=CONTEXT.account_id, + role=TenantAccountRole.OWNER, + current=True, + ), + make_app(name="Source", app_model_config_id="config-1", api_rpm=10, api_rph=100), + ( + config := AppModelConfig( + app_id="app-1", + model=json.dumps( + { + "provider": "openai", + "name": "gpt-4", + "mode": "chat", + "completion_params": {"stop": ["stop"]}, + } + ), + pre_prompt="Hello {{name}}: {{weather}}", + user_input_form=json.dumps([{"text-input": {"variable": "name", "label": "Name"}}]), + external_data_tools=json.dumps( + [ + { + "enabled": True, + "variable": "weather", + "type": "api", + "config": {"api_based_extension_id": "extension-1"}, + } + ] + ), + dataset_configs=json.dumps( + { + "retrieval_model": "multiple", + "top_k": 3, + "datasets": {"datasets": [{"dataset": {"enabled": True, "id": "dataset-1"}}]}, + } + ), + dataset_query_variable="name", + opening_statement="Welcome", + ) + ), + extension, + ] + ) + config.id = "config-1" + return cast(sessionmaker[Session], factory) + + +@pytest.mark.parametrize("mode", [AppMode.CHAT, AppMode.COMPLETION]) +@pytest.mark.parametrize("agent_enabled", [False, True]) +def test_conversion_releases_reads_and_commits_target_once( + sessions: sessionmaker[Session], mode: AppMode, agent_enabled: bool +) -> None: + with sessions.begin() as session: + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + assert source is not None + assert config is not None + source.mode = mode + config.agent_mode = json.dumps({"enabled": agent_enabled, "strategy": "react"}) + original_config = config.to_dict(annotation_reply={"enabled": False}) + active: set[Session] = set() + commits = [] + event.listen(sessions.class_, "after_begin", lambda session, *_: active.add(session)) + + def end_transaction(session: Session, transaction: SessionTransaction) -> None: + if transaction.parent is None: + active.discard(session) + + event.listen(sessions.class_, "after_transaction_end", end_transaction) + event.listen(sessions.class_, "after_commit", lambda session: commits.append(session)) + notifications = [] + + def decrypt(tenant_id: str, token: str) -> str: + assert not active + assert (tenant_id, token) == ("tenant-1", "encrypted") + return "decrypted" + + def created(*, event: AppEvent, account_id: str, backing_agent_id: str | None) -> None: + assert not active + assert len(commits) == 1 + assert (account_id, backing_agent_id) == (CONTEXT.account_id, None) + with sessions() as session: + assert session.scalar(select(Workflow).where(Workflow.app_id == event.id)) is not None + assert session.scalar(select(Site).where(Site.app_id == event.id)) is not None + assert session.scalar(select(InstalledApp).where(InstalledApp.app_id == event.id)) is not None + notifications.append(event) + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=decrypt, + notify_created=created, + ) + new_id = service.convert(CONTEXT, "app-1", {"name": "", "icon_type": "", "icon": "", "icon_background": ""}) + assert len(notifications) == 1 + assert notifications[0].id == new_id + with sessions() as session: + target = session.get(App, new_id) + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + assert target is not None + assert source is not None + assert config is not None + assert target.name == "Source(workflow)" + assert target.mode == (AppMode.WORKFLOW if mode == AppMode.COMPLETION else AppMode.ADVANCED_CHAT) + assert (target.icon_type, target.icon, target.icon_background) == ( + source.icon_type, + source.icon, + source.icon_background, + ) + assert (target.api_rpm, target.api_rph, target.enable_site, target.enable_api) == (10, 100, True, True) + assert (target.created_by, target.maintainer, target.updated_by) == (CONTEXT.account_id,) * 3 + assert source.mode == mode + assert config.to_dict(annotation_reply={"enabled": False}) == original_config + assert session.scalar(select(Workflow.id).where(Workflow.app_id == source.id)) is None + workflow = session.scalar(select(Workflow).where(Workflow.app_id == new_id)) + assert workflow is not None + assert workflow.version == Workflow.VERSION_DRAFT + nodes = json.loads(workflow.graph)["nodes"] + assert [node["id"] for node in nodes] == [ + "start", + "http_request_1", + "code_1", + "knowledge_retrieval", + "llm", + "end" if mode == AppMode.COMPLETION else "answer", + ] + http = nodes[1]["data"] + assert http["authorization"]["config"]["api_key"] == "decrypted" + params = json.loads(http["body"]["data"])["params"] + assert params["app_id"] == source.id + assert params["query"] == ("{{#sys.query#}}" if mode == AppMode.CHAT else "") + assert nodes[3]["data"]["query_variable_selector"] == ( + ["sys", "query"] if mode == AppMode.CHAT else ["start", "name"] + ) + assert "{{#code_1.result#}}" in json.dumps(nodes[4]) + features = json.loads(workflow.features) + if mode == AppMode.CHAT: + assert features["opening_statement"] == "Welcome" + assert "retriever_resource" in features + else: + assert set(features) == {"text_to_speech", "file_upload", "sensitive_word_avoidance"} + + +@pytest.mark.parametrize("failed_record", [InstalledApp, Workflow]) +def test_failed_conversion_rolls_back_entire_target(sessions: sessionmaker[Session], failed_record: type) -> None: + def fail_write(session: Session, *_: object) -> None: + if any(isinstance(row, failed_record) for row in session.new): + raise RuntimeError("write failed") + + event.listen(sessions.class_, "before_flush", fail_write) + notifications = [] + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=lambda **kwargs: notifications.append(kwargs), + ) + with pytest.raises(RuntimeError, match="write failed"): + service.convert(CONTEXT, "app-1", {}) + assert notifications == [] + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + for model in (Workflow, Site, InstalledApp): + assert session.scalar(select(func.count()).select_from(model)) == 0 + + +@pytest.mark.parametrize( + "invalid", ["mode", "missing_config", "foreign_config", "foreign_extension", "missing_extension"] +) +def test_conversion_rejects_invalid_owned_data_before_creating_target( + sessions: sessionmaker[Session], invalid: str +) -> None: + with sessions.begin() as session: + source = session.get(App, "app-1") + config = session.get(AppModelConfig, "config-1") + extension = session.get(APIBasedExtension, "extension-1") + assert source is not None + assert config is not None + assert extension is not None + if invalid == "mode": + source.mode = AppMode.WORKFLOW + elif invalid == "missing_config": + source.app_model_config_id = None + elif invalid == "foreign_config": + config.app_id = "another-app" + elif invalid == "foreign_extension": + extension.tenant_id = "another-tenant" + else: + session.delete(extension) + notifications: list[Mapping[str, object]] = [] + decryptions: list[tuple[object, ...]] = [] + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *args: decryptions.append(args) or "key", + notify_created=lambda **kwargs: notifications.append(kwargs), + ) + with pytest.raises(WorkflowConversionError): + service.convert(CONTEXT, "app-1", {}) + assert notifications == [] + assert decryptions == [] + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 0 + + +def test_conversion_rejects_other_tenant(sessions: sessionmaker[Session]) -> None: + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=lambda *_: "key", + notify_created=lambda **_: None, + ) + with pytest.raises(ConsoleAppNotFoundError): + service.convert(CONTEXT._replace(active_workspace_id="other-tenant"), "app-1", {}) + + +def test_decryption_failure_creates_no_target(sessions: sessionmaker[Session]) -> None: + def fail(*_: str) -> str: + raise RuntimeError("key service unavailable") + + service = WorkflowConversionService( + ConsoleAppRepository(session_factory=sessions), + converter=WorkflowConverter(), + decrypt_token=fail, + notify_created=lambda **_: None, + ) + with pytest.raises(RuntimeError, match="key service unavailable"): + service.convert(CONTEXT, "app-1", {}) + with sessions() as session: + assert session.scalar(select(func.count()).select_from(App)) == 1 + assert session.scalar(select(func.count()).select_from(Workflow)) == 0 diff --git a/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py new file mode 100644 index 00000000000000..1cfef2cf479279 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_generation_read_boundary.py @@ -0,0 +1,131 @@ +"""Generation must release request reads before waiting for a worker's response.""" + +import json +from unittest.mock import Mock, create_autospec + +import pytest +from flask import Flask +from sqlalchemy import inspect +from sqlalchemy.orm import Session + +from core.app.apps import base_app_queue_manager +from core.app.entities.app_invoke_entities import InvokeFrom +from core.ops.ops_trace_manager import TraceQueueManager +from models.enums import EndUserType +from models.model import App, AppMode, EndUser +from models.workflow import Workflow, WorkflowType +from services.app.generation.runtime import AppGenerationRuntime +from services.app_generate_service import AppGenerateService +from services.workflow.execution.adapters.chatflow import app_generator as chatflow_module +from services.workflow.execution.adapters.workflow import app_generator as workflow_module +from services.workflow.variable_contracts import WorkflowExecutionVariables +from tests.unit_tests.services.test_app_generate_service import _DummyRateLimit + + +@pytest.mark.parametrize("mode", [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]) +@pytest.mark.parametrize("fail_while_waiting", [False, True]) +def test_blocking_generation_detaches_request_inputs_before_waiting( + app: Flask, + sqlite_session: Session, + workflow_runtime: AppGenerationRuntime, + workflow_variables: WorkflowExecutionVariables, + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + fail_while_waiting: bool, +) -> None: + application = App( + id="app", + tenant_id="tenant", + name="App", + mode=mode, + description="", + enable_site=False, + enable_api=True, + max_active_requests=None, + ) + user = EndUser( + id="user", + tenant_id="tenant", + app_id="app", + type=EndUserType.SERVICE_API, + session_id="external-user", + ) + workflow = Workflow.new( + tenant_id="tenant", + app_id="app", + type=WorkflowType.CHAT.value if mode == AppMode.ADVANCED_CHAT else WorkflowType.WORKFLOW.value, + version="1", + created_by="owner", + features="{}", + graph=json.dumps( + {"nodes": [{"id": "start", "data": {"type": "start", "title": "Start", "variables": []}}], "edges": []} + ), + environment_variables=[], + conversation_variables=[], + rag_pipeline_variables=[], + ) + application.workflow_id = workflow.id + sqlite_session.add_all([application, user, workflow]) + sqlite_session.commit() + sqlite_session.refresh(application) + assert sqlite_session.in_transaction() + + trace_queue_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + if mode == AppMode.ADVANCED_CHAT: + generator = chatflow_module.AdvancedChatAppGenerator + converter = chatflow_module.AdvancedChatAppGenerateResponseConverter + monkeypatch.setattr(chatflow_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) + else: + generator = workflow_module.WorkflowAppGenerator + converter = workflow_module.WorkflowAppGenerateResponseConverter + monkeypatch.setattr(workflow_module, "TraceQueueManager", Mock(return_value=trace_queue_manager)) + monkeypatch.setattr("services.app_generate_service.RateLimit", _DummyRateLimit) + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + monkeypatch.setattr(generator, "_generate_worker", staticmethod(lambda **_kwargs: None)) + monkeypatch.setattr(converter, "convert", lambda *, response, **_kwargs: response) + waits: list[str] = [] + + def wait_for_response(_self: object, **_kwargs: object) -> dict[str, str]: + assert not sqlite_session.in_transaction() + for row in (application, user, workflow): + assert inspect(row).detached + assert application.name == "App" + assert workflow.graph_dict["nodes"][0]["id"] == "start" + assert not sqlite_session.in_transaction() + waits.append("response") + if fail_while_waiting: + raise ValueError("model response failed") + return {"answer": "done"} + + monkeypatch.setattr( + generator, + "_handle_advanced_chat_response" if mode == AppMode.ADVANCED_CHAT else "_handle_response", + wait_for_response, + ) + with app.app_context(): + if fail_while_waiting: + with pytest.raises(ValueError, match="model response failed"): + AppGenerateService.generate( + app_model=application, + user=user, + args={"query": "Hello", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=sqlite_session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + else: + result = AppGenerateService.generate( + app_model=application, + user=user, + args={"query": "Hello", "inputs": {}}, + invoke_from=InvokeFrom.SERVICE_API, + streaming=False, + session=sqlite_session, + variables=workflow_variables, + runtime=workflow_runtime, + ) + assert result == {"answer": "done"} + assert waits == ["response"] + assert not sqlite_session.in_transaction() diff --git a/api/tests/unit_tests/services/workflow/test_graph_check.py b/api/tests/unit_tests/services/workflow/test_graph_check.py index b61d63d47612e4..48579722cc2c3e 100644 --- a/api/tests/unit_tests/services/workflow/test_graph_check.py +++ b/api/tests/unit_tests/services/workflow/test_graph_check.py @@ -1,9 +1,10 @@ from collections.abc import Mapping import pytest -from sqlalchemy.orm import Session +from sqlalchemy.orm import Session, sessionmaker from models import AppMode +from repositories.credentials.query_repository import CredentialQueryRepository from services.workflow.graph_check import IssueCode, IssueSeverity, check_graph, credential_check from services.workflow_service import WorkflowService @@ -16,6 +17,11 @@ def _node(node_id: str, data: dict[str, object]) -> dict[str, object]: END = _node("end", {"type": "end", "title": "End", "outputs": []}) +@pytest.fixture +def credential_query(sqlite_session_factory: sessionmaker[Session]) -> CredentialQueryRepository: + return CredentialQueryRepository(session_factory=sqlite_session_factory) + + def _template(selector: list[str]) -> dict[str, object]: return _node( "t", @@ -65,9 +71,9 @@ def test_graph_without_nodes() -> None: ], ) def test_malformed_shapes_are_graph_issues_not_errors( - sqlite_session: Session, graph: Mapping[str, object], loc: tuple[object, ...] + credential_query: CredentialQueryRepository, graph: Mapping[str, object], loc: tuple[object, ...] ) -> None: - issues = check_graph(graph, mode=AppMode.WORKFLOW, resources=credential_check("ws-1", {}, sqlite_session)) + issues = check_graph(graph, mode=AppMode.WORKFLOW, resources=credential_check("ws-1", {}, credential_query)) assert [(i.code, i.loc) for i in issues] == [(IssueCode.GRAPH_INVALID, loc)] @@ -226,10 +232,10 @@ def validate(_self: WorkflowService, tenant_id: str, provider: str, model_name: def test_credential_check_reports_what_publish_refuses_for_a_node( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, credential_query: CredentialQueryRepository ) -> None: llm = _node("l", {"type": "llm", "model": {"provider": "openai", "name": "gpt-4o"}}) - check = credential_check("ws-1", {}, sqlite_session) + check = credential_check("ws-1", {}, credential_query) calls = _validate_llm_raising(monkeypatch, ValueError("no credential")) assert check(llm) == "no credential" @@ -237,10 +243,10 @@ def test_credential_check_reports_what_publish_refuses_for_a_node( def test_credential_check_wraps_unexpected_errors_with_the_node( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session + monkeypatch: pytest.MonkeyPatch, credential_query: CredentialQueryRepository ) -> None: llm = _node("l", {"type": "llm", "model": {"provider": "openai", "name": "gpt-4o"}}) - check = credential_check("ws-1", {}, sqlite_session) + check = credential_check("ws-1", {}, credential_query) _validate_llm_raising(monkeypatch, RuntimeError("boom")) assert check(llm) == "Node l (llm): boom" @@ -248,24 +254,24 @@ def test_credential_check_wraps_unexpected_errors_with_the_node( @pytest.mark.parametrize("node_type", ["knowledge-retrieval", "parameter-extractor", "question-classifier"]) def test_credential_check_matches_publish_for_model_nodes_publish_skips( - monkeypatch: pytest.MonkeyPatch, sqlite_session: Session, node_type: str + monkeypatch: pytest.MonkeyPatch, credential_query: CredentialQueryRepository, node_type: str ) -> None: """Publish lists these types with underscores, so it never checks them; the release check must agree.""" node = _node("n", {"type": node_type, "model": {"provider": "openai", "name": "gpt-4o"}}) - check = credential_check("ws-1", {}, sqlite_session) + check = credential_check("ws-1", {}, credential_query) calls = _validate_llm_raising(monkeypatch, ValueError("no credential")) assert check(node) is None assert calls == [] -def test_chatflow_starter_draft_only_lacks_a_model(sqlite_session: Session) -> None: +def test_chatflow_starter_draft_only_lacks_a_model(credential_query: CredentialQueryRepository) -> None: from services.app.console_gateway import _STARTER_DRAFT_GRAPHS issues = check_graph( _STARTER_DRAFT_GRAPHS[AppMode.ADVANCED_CHAT], mode=AppMode.ADVANCED_CHAT, - resources=credential_check("ws-1", {}, sqlite_session), + resources=credential_check("ws-1", {}, credential_query), ) assert [(i.code, i.node_id, i.message) for i in issues] == [ diff --git a/api/tests/unit_tests/services/workflow/test_human_input_persistence.py b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py new file mode 100644 index 00000000000000..fc0a6089c45365 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_human_input_persistence.py @@ -0,0 +1,213 @@ +"""Human Input nodes persist through the execution's injected database.""" + +import json +from collections.abc import Iterator +from datetime import timedelta +from typing import cast + +import pytest +from sqlalchemy import Table, create_engine, event, select +from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import QueuePool + +from core.app.entities.app_invoke_entities import InvokeFrom +from core.db import session_factory +from core.workflow.system_variables import build_system_variables +from enums.human_input import HumanInputFormStatus, RecipientType +from extensions.application_services.workflow import build_workflow_execution_dependencies +from extensions.application_services.workflow_variables import build_workflow_variable_service +from graphon.engine import Engine +from graphon.engine_events import EngineEvent, NodeRunSucceededEvent +from graphon.file.protocols import WorkflowFileRuntimeProtocol +from graphon.node_events import PauseRequestedEvent, StreamCompletedEvent +from graphon.runtime import RuntimeState, VariablePool +from libs.datetime_utils import naive_utc_now +from models.base import TypeBase +from models.human_input import HumanInputDelivery, HumanInputForm, HumanInputFormRecipient +from models.human_input_entities import HumanInputNodeData, UserActionConfig +from services.workflow.execution.adapters.node_factory import DifyNodeFactory +from services.workflow.execution.ports import WorkflowRuntime +from tasks.app_generate.workflow_execute_task import AppExecutionParams, _Account, _AppRunner +from tests.workflow_test_utils import build_test_graph_init_params + +type FormDatabases = tuple[sessionmaker[Session], sessionmaker[Session]] + + +@pytest.fixture(params=[False, True], ids=["retain-after-commit", "expire-after-commit"]) +def form_databases(monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[FormDatabases]: + engines = [create_engine("sqlite://", poolclass=QueuePool) for _ in range(2)] + for engine in engines: + tables: list[Table] = [] + for model in (HumanInputForm, HumanInputDelivery, HumanInputFormRecipient): + table = model.__table__ + assert isinstance(table, Table) + tables.append(table) + TypeBase.metadata.create_all(engine, tables=tables) + injected, global_sessions = [sessionmaker(engine, expire_on_commit=bool(request.param)) for engine in engines] + monkeypatch.setattr(session_factory, "create_session", global_sessions) + try: + yield injected, global_sessions + finally: + for engine in engines: + engine.dispose() + + +def factory(runtime: WorkflowRuntime | None, invoke_from: InvokeFrom = InvokeFrom.DEBUGGER) -> DifyNodeFactory: + state = RuntimeState( + workflow_id="workflow", + variable_pool=VariablePool.from_bootstrap( + system_variables=build_system_variables(app_id="app", workflow_execution_id="run") + ), + start_at=0, + ) + return DifyNodeFactory( + init_params=build_test_graph_init_params(invoke_from=invoke_from), + runtime_state=state, + workflow_runtime=runtime, + ) + + +def node_config() -> dict[str, object]: + return { + "id": "human-node", + "data": HumanInputNodeData( + title="Approval", + form_content="Approve this request?", + user_actions=[UserActionConfig(id="approve", title="Approve")], + ).model_dump(), + } + + +@pytest.mark.parametrize("invoke_from", [InvokeFrom.DEBUGGER, InvokeFrom.SERVICE_API]) +@pytest.mark.parametrize("rebind", [False, True]) +@pytest.mark.parametrize("entry", ["factory", "engine-worker"]) +def test_node_creates_and_resumes_form_in_injected_database( + form_databases: FormDatabases, invoke_from: InvokeFrom, rebind: bool, entry: str +) -> None: + sessions, global_sessions = form_databases + if entry == "engine-worker": + runner = _AppRunner( + sessions.kw["bind"], + AppExecutionParams( + app_id="app", + workflow_id="workflow", + tenant_id="tenant", + user=_Account(user_id="user"), + args={}, + invoke_from=invoke_from, + ), + variables=build_workflow_variable_service(database_client=sessions), + ) + assert runner._session_factory.kw["expire_on_commit"] is True + runtime = runner._runtime + else: + runtime = build_workflow_execution_dependencies(sessions) + node_factory = factory(runtime, invoke_from) + if rebind: + node_factory = node_factory.with_runtime_state(node_factory.runtime_state) + node = node_factory.create_node(node_config()) + node.bind_execution_id("human-execution") + events = list(node._run()) + assert any(isinstance(event, PauseRequestedEvent) for event in events) + assert sessions.kw["bind"].pool.checkedout() == 0 + assert global_sessions.kw["bind"].pool.checkedout() == 0 + + with sessions.begin() as session: + form = session.scalars(select(HumanInputForm)).one() + assert (form.tenant_id, form.app_id, form.workflow_run_id, form.node_id) == ( + "tenant", + "app", + "run", + "human-node", + ) + form_id = form.id + recipients = session.scalars(select(HumanInputFormRecipient)).all() + if invoke_from == InvokeFrom.DEBUGGER: + console = next(recipient for recipient in recipients if recipient.recipient_type == RecipientType.CONSOLE) + assert json.loads(console.recipient_payload)["account_id"] == "user" + form.selected_action_id = "approve" + form.submitted_data = "{}" + form.submitted_at = naive_utc_now() + form.expiration_time = naive_utc_now() + timedelta(hours=1) + form.status = HumanInputFormStatus.SUBMITTED + + completed = list(node._run()) + assert not any(isinstance(event, PauseRequestedEvent) for event in completed) + assert isinstance(completed[-1], StreamCompletedEvent) + assert completed[-1].node_run_result.outputs["__action_id"].value == "approve" + with sessions() as session: + assert session.scalars(select(HumanInputForm)).one().id == form_id + with global_sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert sessions.kw["bind"].pool.checkedout() == 0 + + +def test_human_input_node_requires_persistence_dependency() -> None: + with pytest.raises(ValueError, match="execution dependencies are required for Human Input"): + factory(None).create_node(node_config()) + + +def test_create_form_commit_failure_does_not_return_a_pause_or_persist_form( + form_databases: FormDatabases, +) -> None: + sessions, _ = form_databases + node = factory(build_workflow_execution_dependencies(sessions)).create_node(node_config()) + node.bind_execution_id("human-execution") + + def fail_commit(_session: Session) -> None: + raise RuntimeError("commit failed") + + event.listen(sessions, "before_commit", fail_commit) + try: + with pytest.raises(RuntimeError, match="commit failed"): + list(node._run()) + finally: + event.remove(sessions, "before_commit", fail_commit) + with sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert session.scalar(select(HumanInputFormRecipient)) is None + assert session.scalar(select(HumanInputDelivery)) is None + + +def test_created_form_timeout_reaches_event_stream_in_same_database(form_databases: FormDatabases) -> None: + from dataclasses import dataclass + + from graphon.engine_events import NodeRunHumanInputFormTimeoutEvent + from graphon.graph import Graph + from services.workflow.execution.adapters.workflow_entry import iter_dify_graph_engine_events + + sessions, global_sessions = form_databases + runtime = build_workflow_execution_dependencies(sessions) + node = factory(runtime).create_node(node_config()) + node.bind_execution_id("timed-out-execution") + assert any(isinstance(event, PauseRequestedEvent) for event in node._run()) + deadline = naive_utc_now() - timedelta(minutes=1) + with sessions.begin() as session: + form = session.scalars(select(HumanInputForm)).one() + form.expiration_time = deadline + form.status = HumanInputFormStatus.TIMEOUT + + @dataclass + class NodeEvents: + graph: Graph + runtime_state: RuntimeState + file_runtime: WorkflowFileRuntimeProtocol | None = None + + def run(self) -> Iterator[EngineEvent]: + for run_event in node.run(): + assert isinstance(run_event, EngineEvent) + yield run_event + + events = list( + iter_dify_graph_engine_events( + cast(Engine, NodeEvents(Graph(root_node=node), node.runtime_state)), + human_form_reader=runtime.human_form_reader, + ) + ) + timeout = next(event for event in events if isinstance(event, NodeRunHumanInputFormTimeoutEvent)) + assert timeout.id == "timed-out-execution" + assert timeout.expiration_time == deadline + assert isinstance(events[-1], NodeRunSucceededEvent) + with global_sessions() as session: + assert session.scalar(select(HumanInputForm)) is None + assert sessions.kw["bind"].pool.checkedout() == 0 diff --git a/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py b/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py index 3ec90c25966613..5fc0b922829d52 100644 --- a/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py +++ b/api/tests/unit_tests/services/workflow/test_node_output_inspector_service.py @@ -16,17 +16,9 @@ from sqlalchemy.orm import Session from core.workflow.file_reference import build_file_reference -from core.workflow.nodes.agent_v2.binding_resolver import WorkflowAgentBindingBundle +from enums.agent import WorkflowAgentBindingType from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus -from models.agent import ( - Agent, - AgentConfigSnapshot, - AgentScope, - AgentSource, - AgentStatus, - WorkflowAgentBindingType, - WorkflowAgentNodeBinding, -) +from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding from models.agent_config_entities import ( AgentSoulConfig, DeclaredArrayItem, @@ -34,6 +26,7 @@ DeclaredOutputType, WorkflowNodeJobConfig, ) +from models.agent_runtime_contracts import WorkflowAgentBindingBundle from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.model import App from models.workflow import ( diff --git a/api/tests/unit_tests/services/workflow/test_publication_service.py b/api/tests/unit_tests/services/workflow/test_publication_service.py new file mode 100644 index 00000000000000..fb89962ef235b2 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_publication_service.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from dataclasses import replace +from unittest.mock import Mock, create_autospec + +from repositories.workflow.definition_repository import workflow_snapshot +from services.agent.publication_contracts import AgentPublicationState +from services.workflow.contracts import ( + PublicationConfiguration, + WorkflowBindingOperations, + WorkflowPublicationTransaction, +) +from services.workflow.publication_service import publish_workflow +from tests.helpers.unit.model_factories import make_workflow + + +def test_publish_syncs_knowledge_spaces_after_agent_bindings_and_before_activation() -> None: + draft = workflow_snapshot(make_workflow(workflow_id="draft")) + published = replace(draft, id="published", version="1") + transaction = create_autospec(WorkflowPublicationTransaction, instance=True, spec_set=True) + transaction.draft = draft + transaction.create_version.return_value = published + agents: AgentPublicationState = () + agent_service = create_autospec(WorkflowBindingOperations, instance=True, spec_set=True) + calls = Mock() + calls.attach_mock(agent_service, "agents") + calls.attach_mock(transaction, "transaction") + + result = publish_workflow( + transaction, + PublicationConfiguration( + dataset_ids={"dataset"}, + webhook_nodes=["webhook"], + plugins=[{"provider": "plugin"}], + schedule=None, + triggers=[{"node_id": "trigger"}], + has_trigger_runtime=True, + ), + agent_service=agent_service, + agents=agents, + marked_name="Release", + marked_comment="Ready", + ) + + assert result is published + transaction.sync_knowledge_spaces.assert_called_once_with(published) + call_names = [entry[0] for entry in calls.mock_calls] + assert call_names.index("agents.copy_agent_node_bindings_to_published") < call_names.index( + "transaction.sync_knowledge_spaces" + ) + assert call_names.index("transaction.sync_knowledge_spaces") < call_names.index("transaction.activate") diff --git a/api/tests/unit_tests/services/workflow/test_runtime_gateway.py b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py new file mode 100644 index 00000000000000..a0dea0e83416a5 --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_runtime_gateway.py @@ -0,0 +1,2196 @@ +"""Runtime adaptation releases read transactions before external execution preparation.""" + +import json +from collections.abc import Callable, Generator, Iterable +from dataclasses import replace +from typing import Literal, NoReturn, Self, cast, override +from unittest.mock import Mock, create_autospec + +import pytest +from flask import Flask +from sqlalchemy import event, select +from sqlalchemy.orm import Session, sessionmaker + +from configs import dify_config +from core.app.entities.app_invoke_entities import WorkflowAppGenerateEntity +from core.app.file_access import DatabaseFileAccessController +from core.repositories.sqlalchemy_workflow_node_execution_repository import SQLAlchemyWorkflowNodeExecutionRepository +from core.trigger.debug.event_selectors import TriggerDebugEvent +from extensions.application_services.workflow import ( + WorkflowExecutionDependencies, + build_workflow_execution_dependencies, +) +from extensions.ext_redis import RedisClientWrapper +from extensions.ext_storage import storage +from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse +from graphon.entities import WorkflowNodeExecution +from graphon.file import File +from machinery.context import RequestContext +from models import Account, EndUser +from models.account import TenantAccountJoin, TenantAccountRole +from models.enums import CreatorUserRole, ExecutionOffLoadType +from models.model import AppMode +from models.workflow import ( + Workflow, + WorkflowDraftVariable, + WorkflowNodeExecutionModel, + WorkflowNodeExecutionOffload, + WorkflowNodeExecutionStatus, + WorkflowNodeExecutionTriggeredFrom, +) +from repositories.agent.retirement_repository import WorkflowAgentRetirementRepository +from repositories.factory import DifyAPIRepositoryFactory +from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository +from repositories.workflow.definition_repository import WorkflowDefinitionRepository, workflow_snapshot +from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository +from repositories.workflow.node_execution_repository import WorkflowNodeExecutionRepository +from services import workflow_service as workflow_module +from services.agent.workflow_publish_service import WorkflowAgentPublishService +from services.app_generate_service import AppGenerateService +from services.app_task_service import AppTaskControlService +from services.file_service import FileService +from services.tools.provider_queries import ToolProviderIcons +from services.workflow import runtime_gateway as module +from services.workflow.console_service import ( + ConsoleWorkflowService, + WorkflowAccess, + WorkflowAppLookup, + WorkflowConversion, + WorkflowDefinitionLifecycle, + WorkflowDefinitions, + WorkflowPresence, +) +from services.workflow.draft_service import WorkflowDraftService +from services.workflow.runtime_gateway import WorkflowRuntimeGateway +from services.workflow.variable_contracts import WorkflowExecutionVariables +from services.workflow.variable_file_gateway import WorkflowVariableFileGateway +from services.workflow.variable_service import WorkflowVariableService +from services.workflow_service import WorkflowService +from tests.helpers.unit.config_override import apply_config_overrides +from tests.helpers.unit.model_factories import make_account, make_app, make_tenant, make_workflow +from tests.unit_tests.workflow_execution import debug_lease, set_debug_deadline + +CONTEXT = RequestContext("request", None, "account-1", "tenant-1") + + +class RecordingSession(Session): + closed = False + + @override + def close(self) -> None: + self.closed = True + super().close() + + +@pytest.fixture(name="runtime") +def runtime_gateway( + sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables +) -> tuple[WorkflowRuntimeGateway, list[RecordingSession]]: + with sqlite_session_factory.begin() as session: + session.add_all( + [ + make_app(mode=AppMode.WORKFLOW), + make_account(), + make_tenant(), + make_workflow(), + TenantAccountJoin(tenant_id="tenant-1", account_id="account-1", role=TenantAccountRole.OWNER), + ] + ) + sessions: list[RecordingSession] = [] + factory = cast( + sessionmaker[Session], + sessionmaker(bind=sqlite_session_factory.kw["bind"], class_=RecordingSession, expire_on_commit=False), + ) + + def opened(session: Session, _transaction: object, _connection: object) -> None: + assert isinstance(session, RecordingSession) + sessions.append(session) + + event.listen(factory, "after_begin", opened) + workflows = WorkflowService(session_maker=factory, runtime=build_workflow_execution_dependencies(factory)) + + def generate(**_kwargs: object) -> Generator[str]: + assert sessions + assert all(session.closed for session in sessions) + + def events() -> Generator[str]: + assert all(session.closed for session in sessions) + yield "data: ready\n\n" + + return events() + + generator = create_autospec(AppGenerateService, spec_set=True) + generator.generate_workflow_stream.side_effect = generate + gateway = WorkflowRuntimeGateway( + runtime=build_workflow_execution_dependencies(factory), + session_factory=factory, + workflows=workflows, + definitions=WorkflowDefinitionRepository(session_factory=factory), + reservations=WorkflowDebugReservationRepository(factory), + executions=WorkflowNodeExecutionRepository(factory), + generator=generator, + tasks=AppTaskControlService( + redis_client=create_autospec(RedisClientWrapper, instance=True, spec_set=True), + ), + file_access=DatabaseFileAccessController(), + variables=workflow_variables, + ) + return gateway, sessions + + +def test_generate_closes_session_before_stream_is_consumed( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], +) -> None: + gateway, sessions = runtime + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert all(session.closed for session in sessions) + assert list(stream) == ["data: ready\n\n"] + + +def test_stop_scopes_task_to_the_requested_app( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, _ = runtime + stop = Mock(return_value=True) + monkeypatch.setattr(gateway._tasks, "stop_workflow_task", stop) + + gateway.stop(CONTEXT, "app-1", "task-1") + + stop.assert_called_once_with( + tenant_id="tenant-1", + app_id="app-1", + task_id="task-1", + app_mode=AppMode.WORKFLOW, + owner=None, + ) + + +@pytest.mark.parametrize("rejection", ["expired", "claimed", "paused", "tenant", "app", "workflow", "mode"]) +def test_rejected_debug_handoff_only_notifies_owned_expired_stream( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + rejection: str, +) -> None: + from datetime import timedelta + + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowRun + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + reservation = debug_lease(session, snapshot.execution_id) + assert run is not None + assert reservation is not None + if rejection in ("claimed", "paused"): + WorkflowDebugReservationRepository.claim(session, run, naive_utc_now()) + if rejection == "paused": + run.status = WorkflowExecutionStatus.PAUSED + set_debug_deadline(session, snapshot.execution_id, naive_utc_now() - timedelta(seconds=30)) + if rejection in ("tenant", "app", "workflow"): + params = params.model_copy(update={f"{rejection}_id": "wrong-owner"}) + elif rejection == "mode": + params = params.model_copy(update={"invoke_from": InvokeFrom.SERVICE_API}) + topic = Mock() + + def publish(_payload: bytes) -> None: + assert all(session.closed and not session.in_transaction() for session in sessions) + + topic.publish.side_effect = publish + get_topic = Mock(return_value=topic) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", get_topic) + finish = Mock(wraps=WorkflowAgentRetirementService.finish_execution) + monkeypatch.setattr(WorkflowAgentRetirementService, "finish_execution", finish) + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + engine = Mock() + monkeypatch.setattr(runner, "_run_app", engine) + with pytest.raises( + ValueError, + match=( + r"Trigger debug reservation (?:expired before worker startup|was already claimed)" + r"|Workflow snapshot does not belong to this debugger invocation" + ), + ): + runner.run() + engine.assert_not_called() + finish.assert_not_called() + if rejection == "expired": + payloads = [json.loads(call.args[0]) for call in topic.publish.call_args_list] + assert [payload["event"] for payload in payloads] == ["workflow_started", "workflow_finished"] + assert payloads[-1]["data"]["status"] == "failed" + assert payloads[-1]["workflow_run_id"] == snapshot.execution_id + assert "expired before worker startup" in payloads[-1]["data"]["error"] + else: + get_topic.assert_not_called() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == ( + WorkflowExecutionStatus.PAUSED if rejection == "paused" else WorkflowExecutionStatus.RUNNING + ) + + +def test_claimed_worker_is_not_cancelled_by_stream_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from datetime import timedelta + + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from services.agent.retirement_service import WorkflowAgentRetirementService + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + WorkflowDebugReservationRepository.claim(session, run, naive_utc_now()) + gateway._cancel_trigger_debug(snapshot, CONTEXT.account_id) + # Even an expiry candidate selected before the claim must be rechecked + # under the run lock; a healthy running lease must remain owned. + assert ( + WorkflowDebugReservationRepository(sqlite_session_factory).expire( + snapshot.execution_id, naive_utc_now() + timedelta(seconds=1) + ) + is None + ) + with sqlite_session_factory() as session: + assert session.get(WorkflowRun, snapshot.execution_id) is not None + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("core.db.session_factory._session_maker", sessionmaker()) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + WorkflowAgentRetirementService.finish_execution( + sessions=sqlite_session_factory, + tenant_id="tenant-1", + app_id="app-1", + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=CONTEXT.account_id, + status=WorkflowExecutionStatus.STOPPED, + ) + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + collector.assert_called_once() + + +@pytest.mark.parametrize("storage_backend", ["sqlalchemy", "celery"]) +@pytest.mark.parametrize("stream_raises", [False, True]) +@pytest.mark.parametrize("publish_fails", [False, True]) +def test_paused_trigger_finalizes_once_despite_delayed_start_write( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + storage_backend: str, + stream_raises: bool, + publish_fails: bool, +) -> None: + from contextlib import contextmanager + + from core.app.entities.app_invoke_entities import InvokeFrom + from core.repositories.celery_workflow_execution_repository import CeleryWorkflowExecutionRepository + from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchemyWorkflowExecutionRepository + from graphon.entities import WorkflowExecution + from graphon.enums import WorkflowExecutionStatus, WorkflowType + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.enums import WorkflowRunTriggeredFrom + from models.workflow import WorkflowRun + from repositories.agent.runtime_repository import WorkflowAgentExecutionRepository + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks import workflow_execution_tasks as storage_module + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + started = WorkflowExecution( + id_=snapshot.execution_id, + workflow_id=snapshot.id, + workflow_type=WorkflowType.WORKFLOW, + workflow_version="draft", + graph=snapshot.graph_dict, + inputs={}, + outputs={}, + status=WorkflowExecutionStatus.RUNNING, + error_message="", + total_tokens=0, + total_steps=0, + exceptions_count=0, + started_at=snapshot.created_at, + finished_at=None, + ) + paused = started.model_copy(update={"status": WorkflowExecutionStatus.PAUSED}) + repository_type = ( + CeleryWorkflowExecutionRepository if storage_backend == "celery" else SQLAlchemyWorkflowExecutionRepository + ) + writer = repository_type( + session_factory=sqlite_session_factory, + tenant_id="tenant-1", + app_id="app-1", + user=actor, + triggered_from=WorkflowRunTriggeredFrom.DEBUGGING, + ) + monkeypatch.setattr("core.db.session_factory.session_factory.create_session", gateway._sessions) + queued = Mock() + monkeypatch.setattr(storage_module.save_workflow_execution_task, "delay", queued) + if storage_backend == "celery": + writer.save(started) + writer.save(paused) + assert queued.call_count == 2 + + def deliver(index: int) -> None: + if storage_backend == "celery": + storage_module.save_workflow_execution_task(**queued.call_args_list[index].kwargs) + else: + writer.save((started, paused)[index]) + + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + from repositories.workflow.execution_write_repository import WorkflowExecutionWriteRepository + + statuses: list[WorkflowExecutionStatus | None] = [] + original_finish = WorkflowExecutionWriteRepository.finish + + def record_finish( + self: WorkflowExecutionWriteRepository, + *, + tenant_id: str, + app_id: str, + workflow_id: str, + execution_id: str, + status: WorkflowExecutionStatus | None = None, + cancel: bool = False, + ) -> WorkflowExecutionStatus | None: + statuses.append(status) + return original_finish( + self, + tenant_id=tenant_id, + app_id=app_id, + workflow_id=workflow_id, + execution_id=execution_id, + status=status, + cancel=cancel, + ) + + monkeypatch.setattr(WorkflowExecutionWriteRepository, "finish", record_finish) + topic = Mock() + if publish_fails: + topic.publish.side_effect = RuntimeError("pause publish failed") + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: topic) + + def events(**_kwargs: object) -> Generator[dict[str, object]]: + yield {"event": "workflow_paused", "data": {"status": "paused"}} + if stream_raises: + raise RuntimeError("stream failed after pause") + + @contextmanager + def execution_context(_user: object) -> Generator[None]: + try: + yield + finally: + # The publisher has finalized PAUSED, but the outer run() has not + # exited yet. Deliver the older asynchronous start write here. + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + deliver(0) + + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", events) + monkeypatch.setattr(runner, "_setup_flask_context", execution_context) + if publish_fails: + with pytest.raises(RuntimeError, match="pause publish failed"): + runner.run() + assert [json.loads(call.args[0])["event"] for call in topic.publish.call_args_list] == [ + "workflow_paused", + "workflow_paused", + ] + elif stream_raises: + with pytest.raises(RuntimeError, match="stream failed after pause"): + runner.run() + else: + runner.run() + + assert statuses == [WorkflowExecutionStatus.PAUSED] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + + # Finally deliver the asynchronous pause write. The outer runner must not + # have poisoned the execution with FAILED or retired its removed Agent. + deliver(1) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.PAUSED + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + assert WorkflowAgentExecutionRepository.retained_execution_agent_ids(session, "tenant-1", ["agent-1"]) == { + "agent-1" + } + collector.assert_not_called() + + +@pytest.mark.parametrize("stream_raises", [False, True]) +def test_pause_publication_can_resume_before_old_worker_exits( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + stream_raises: bool, +) -> None: + from contextlib import nullcontext + + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowPause, WorkflowRun + from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.execution_write_repository import save_workflow_run + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + runs = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory) + + def events(**_kwargs: object) -> Generator[dict[str, object]]: + # Pause-state persistence precedes the outgoing engine event. + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + run.status = WorkflowExecutionStatus.PAUSED + session.add(WorkflowPause(workflow_id=workflow.id, workflow_run_id=run.id, state_object_key="pause")) + yield {"event": "workflow_paused", "data": {"status": "paused"}} + if stream_raises: + raise RuntimeError("old stream failed after resume") + + def resume_on_publication(_payload: bytes) -> None: + pause = runs.get_workflow_pause(snapshot.execution_id) + assert pause is not None + runs.resume_workflow_pause(snapshot.execution_id, pause) + + topic = Mock() + topic.publish.side_effect = resume_on_publication + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: topic) + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", events) + monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) + if stream_raises: + with pytest.raises(RuntimeError, match="old stream failed after resume"): + runner.run() + else: + runner.run() + topic.publish.assert_called_once() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert agent is not None + assert agent.status == AgentStatus.ACTIVE + session.expunge(run) + with sqlite_session_factory.begin() as session: + assert save_workflow_run(session, run) + collector.assert_not_called() + # A subsequently lost resumed worker must still be discoverable by expiry. + from datetime import timedelta + + assert snapshot.execution_id in [ + item.execution_id + for item in WorkflowDebugReservationRepository(sqlite_session_factory).pending_batch( + naive_utc_now() + timedelta(days=1), limit=100 + ) + ] + + +@pytest.mark.parametrize("failure_stage", ["prepare", "publish-setup"]) +def test_trigger_runner_finalizes_failure_before_stream_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + failure_stage: str, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("core.db.session_factory.session_factory.create_session", gateway._sessions) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + finish = Mock(wraps=WorkflowAgentRetirementService.finish_execution) + monkeypatch.setattr(WorkflowAgentRetirementService, "finish_execution", finish) + topic = Mock() + if failure_stage == "publish-setup": + topic.side_effect = RuntimeError("stream setup failed") + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", topic) + generate = Mock(return_value=(event for event in [])) + if failure_stage == "prepare": + generate.side_effect = RuntimeError("stream setup failed") + runner = task_module._AppRunner(gateway._sessions, params, variables=gateway._variables) + monkeypatch.setattr(runner, "_run_app", generate) + + with Flask(__name__).app_context(), pytest.raises(RuntimeError, match="stream setup failed"): + runner.run() + + finish.assert_called_once() + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert agent is not None + assert run.status == WorkflowExecutionStatus.FAILED + assert agent.status == AgentStatus.ARCHIVED + collector.assert_called_once() + + +@pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +@pytest.mark.parametrize("failure", [None, "redis"]) +def test_real_stream_preparation_releases_reads_before_external_io( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + failure: str | None, +) -> None: + from core.app.features.rate_limiting import rate_limit as limiter_module + from enums import DeploymentEdition + from models.model import App + from services.quota_service import QuotaCharge, QuotaService, unlimited + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = runtime + gateway._generator = AppGenerateService + with sqlite_session_factory.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = mode + app.max_active_requests = 2 + apply_config_overrides( + monkeypatch, + DEPLOYMENT_EDITION=DeploymentEdition.CLOUD, + PUBSUB_REDIS_CHANNEL_TYPE="streams", + ) + monkeypatch.setattr(limiter_module.RateLimit, "_instance_dict", {}) + observed: list[str] = [] + + def external(stage: str) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + observed.append(stage) + if failure == stage: + raise RuntimeError("external failure") + + def reserve(*_args: object) -> QuotaCharge: + external("quota") + return unlimited() + + def redis_write(*_args: object) -> None: + external("redis") + + monkeypatch.setattr(QuotaService, "reserve", reserve) + monkeypatch.setattr(limiter_module.redis_client, "setex", redis_write) + monkeypatch.setattr(limiter_module.redis_client, "exists", lambda _key: False) + monkeypatch.setattr(limiter_module.redis_client, "hlen", lambda _key: 0) + monkeypatch.setattr(limiter_module.redis_client, "hset", redis_write) + monkeypatch.setattr(limiter_module.redis_client, "hdel", redis_write) + queued: list[str] = [] + + def enqueue(payload: str) -> None: + external("enqueue") + queued.append(payload) + + def retrieve_events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str]: + # This assertion runs during preparation, before the returned stream is consumed. + external("subscription-prepare") + + def events() -> Generator[str]: + on_subscribe() + yield from () + + return events() + + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", enqueue) + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", retrieve_events) + if failure: + with Flask(__name__).app_context(), pytest.raises(RuntimeError, match="external failure"): + gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert queued == [] + assert observed == ["quota", "redis"] + else: + with Flask(__name__).app_context(): + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id=None, workflow=None) + assert observed[0] == "quota" + assert "redis" in observed + assert observed[-1] == "subscription-prepare" + assert not queued + assert list(stream) == [] + assert len(queued) == 1 + payload = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert (payload.tenant_id, payload.app_id, payload.app_mode) == ("tenant-1", "app-1", mode) + assert all(session.closed and not session.in_transaction() for session in sessions) + + +def test_trigger_poll_runs_outside_database_session( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, sessions = runtime + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + return TriggerDebugEvent(node_id="node", workflow_args={"inputs": {}}) + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + result = gateway.poll_trigger(CONTEXT, "app-1", ["node"], single_node=False, select_all=False) + assert result is not None + assert result.node_id == "node" + + +def test_schedule_single_node_does_not_poll( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, _ = runtime + with sqlite_session_factory.begin() as session: + workflow = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert workflow is not None + workflow.graph = json.dumps({"nodes": [{"id": "timer", "data": {"type": "trigger-schedule"}}], "edges": []}) + + def unexpected(**_kwargs: object) -> None: + pytest.fail("schedule single-step must not wait for an event") + + monkeypatch.setattr(module, "create_event_poller", unexpected) + result = gateway.poll_trigger(CONTEXT, "app-1", ["timer"], single_node=True, select_all=False) + assert result is not None + assert result.workflow_args == {} + + +def test_node_response_materializes_actor_extras_and_offload_flags( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, sessions = runtime + execution = WorkflowNodeExecutionModel( + id="execution", + tenant_id="tenant-1", + app_id="app-1", + node_id="node", + node_type="start", + created_by="account-1", + created_by_role=CreatorUserRole.ACCOUNT, + status=WorkflowNodeExecutionStatus.SUCCEEDED, + inputs='{"query":"value"}', + outputs="{}", + execution_metadata=None, + ) + execution.offload_data = [] + monkeypatch.setattr(gateway._workflows, "run_draft_workflow_node", lambda **_kwargs: execution) + + def extras(_execution: WorkflowNodeExecutionModel, *, tool_providers: ToolProviderIcons) -> dict[str, object]: + assert tool_providers is gateway._runtime.tool_providers + assert all(session.closed for session in sessions) + return {} + + monkeypatch.setattr("services.workflow.runtime_gateway.node_execution_extras", extras) + result = gateway.run_node(CONTEXT, "app-1", "node", {"inputs": {}}, include_details=True, workflow=None) + assert all(session.closed for session in sessions) + response = WorkflowRunNodeExecutionResponse.model_validate(result).model_dump(mode="json") + assert response["inputs"] == {"query": "value"} + assert response["created_by_account"]["id"] == "account-1" + assert response["extras"] == {} + assert response["inputs_truncated"] is False + raw = gateway.run_node(CONTEXT, "app-1", "node", {"inputs": {}}, include_details=False, workflow=None) + assert raw["inputs"] == '{"query":"value"}' + assert raw["offload_data"] == [] + assert "created_by_account" not in raw + + +def test_trigger_response_preserves_all_offload_types( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + gateway, sessions = runtime + records = [ + WorkflowNodeExecutionOffload( + tenant_id="tenant-1", + app_id="app-1", + node_execution_id="execution", + type_=kind, + file_id=f"file-{kind.value}", + ) + for kind in ExecutionOffLoadType + ] + with sqlite_session_factory.begin() as session: + session.add_all(records) + # Query and detach actual ORM rows so column names and attribute names differ + # exactly as in the runtime, without resolving file relationships. + with sqlite_session_factory() as session: + offloads = list(session.scalars(select(WorkflowNodeExecutionOffload))) + execution = WorkflowNodeExecutionModel(id="execution") + execution.offload_data = offloads + raw = gateway._execution(execution, include_details=False) + assert {item["type_"] for item in raw["offload_data"]} == {"inputs", "outputs", "process_data"} + assert raw["offload_data"] == [ + { + "id": item.id, + "tenant_id": "tenant-1", + "app_id": "app-1", + "node_execution_id": "execution", + "type_": item.type_.value, + "file_id": item.file_id, + "created_at": item.created_at, + } + for item in offloads + ] + assert all(session.closed for session in sessions) + + +def test_file_config_and_access_controller_are_reused( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], monkeypatch: pytest.MonkeyPatch +) -> None: + gateway, _ = runtime + workflow = workflow_snapshot(make_workflow()) + monkeypatch.setattr(module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: None) + assert gateway._files(workflow, [{"id": "file"}]) == [] + config = object() + monkeypatch.setattr(module.FileUploadConfigManager, "convert", lambda *_args, **_kwargs: config) + received = [] + + def build(**kwargs: object) -> list[File]: + received.append(kwargs) + return [] + + monkeypatch.setattr(module.file_factory, "build_from_mappings", build) + gateway._files(workflow, [{"id": "file"}]) + assert received == [ + { + "mappings": [{"id": "file"}], + "tenant_id": "tenant-1", + "config": config, + "access_controller": gateway._file_access, + "sessions": gateway._sessions, + } + ] + + +@pytest.mark.parametrize("concurrent_change", ["configuration", "delete-node"]) +def test_trigger_consumption_and_real_execution_share_detached_revision( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + concurrent_change: str, +) -> None: + gateway, sessions = runtime + + graph: dict[str, object] = { + "nodes": [ + { + "id": "trigger", + "data": { + "type": "trigger-plugin", + "title": "Subscription A", + "plugin_id": "test/trigger", + "provider_id": "test/trigger/provider", + "subscription_id": "subscription-A", + "event_name": "event-A", + "plugin_unique_identifier": "test/trigger:1", + }, + } + ], + "edges": [], + } + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps(graph) + + class ForbiddenDatabase: + @property + def engine(self) -> None: + pytest.fail("single-step execution must use the injected session factory") + + monkeypatch.setattr(workflow_module, "db", ForbiddenDatabase()) + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + changed = json.loads(draft.graph) + if concurrent_change == "delete-node": + changed["nodes"] = [] + else: + changed["nodes"][0]["data"].update( + subscription_id="subscription-B", event_name="event-B", title="Subscription B" + ) + draft.graph = json.dumps(changed) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "from-A"}}) + + def poller(*, draft_workflow: Workflow, **_kwargs: object) -> Poller: + assert draft_workflow.graph_dict["nodes"][0]["data"]["subscription_id"] == "subscription-A" + return Poller() + + monkeypatch.setattr(module, "create_event_poller", poller) + service = ConsoleWorkflowService( + conversion=create_autospec(WorkflowConversion, instance=True, spec_set=True), + agent_services=WorkflowAgentPublishService, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + definitions=create_autospec(WorkflowDefinitions, instance=True, spec_set=True), + lifecycle=create_autospec(WorkflowDefinitionLifecycle, instance=True, spec_set=True), + runtime=gateway, + apps=create_autospec(WorkflowAppLookup, instance=True, spec_set=True), + presence=create_autospec(WorkflowPresence, instance=True, spec_set=True), + access=create_autospec(WorkflowAccess, instance=True, spec_set=True), + ) + result = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=True, select_all=False) + assert isinstance(result, dict) + assert result["status"] == WorkflowNodeExecutionStatus.SUCCEEDED + assert result["title"] == "Subscription A" + assert json.loads(result["outputs"])["payload"] == "from-A" + assert json.loads(result["execution_metadata"])["trigger_info"]["event_name"] == "event-A" + assert all(session.closed for session in sessions) + with sqlite_session_factory() as session: + execution = session.get(WorkflowNodeExecutionModel, result["id"]) + assert execution is not None + assert execution.title == "Subscription A" + variables = session.scalars( + select(WorkflowDraftVariable).where(WorkflowDraftVariable.node_id == "trigger") + ).all() + assert any(variable.name == "payload" and variable.get_value().value == "from-A" for variable in variables) + + +def test_single_node_reads_offloaded_outputs_after_repository_session_closes( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = runtime + content = "complete output " * 1000 + objects: dict[str, bytes] = {} + reads: list[str] = [] + apply_config_overrides( + monkeypatch, + WORKFLOW_VARIABLE_TRUNCATION_MAX_SIZE=256, + WORKFLOW_VARIABLE_TRUNCATION_STRING_LENGTH=128, + ) + monkeypatch.setattr(storage, "save", lambda key, data: objects.__setitem__(key, data)) + + def load(key: str) -> bytes: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + reads.append(key) + return objects[key] + + class IsolatedStorage: + def load(self, filename: str) -> bytes: + return load(filename) + + def delete(self, filename: str) -> None: + objects.pop(filename, None) + + def unexpected_global_read(_key: str) -> bytes: + pytest.fail("single-node execution must use the injected output reader") + + monkeypatch.setattr(storage, "load", unexpected_global_read) + gateway._variables = WorkflowVariableService( + file_inputs=WorkflowVariableFileGateway(sqlite_session_factory, DatabaseFileAccessController()), + repository=WorkflowDraftVariableRepository(sessions=sqlite_session_factory), + files=FileService(sqlite_session_factory), + executions=DifyAPIRepositoryFactory.create_api_workflow_node_execution_repository(sqlite_session_factory), + storage=IsolatedStorage(), + defer_file_cleanup=lambda _ids: None, + ) + + class OffloadedExecutionRepository(SQLAlchemyWorkflowNodeExecutionRepository): + @override + def save(self, execution: WorkflowNodeExecution) -> None: + super().save(execution) + # Keep the real persistence and offload implementation. The reader must + # support a writer that stores the execution payload out of line. + self.save_execution_data(execution) + + runtime_dependencies = gateway._workflows._runtime + assert isinstance(runtime_dependencies, WorkflowExecutionDependencies) + + def node_writer( + *, + tenant_id: str, + user: Account | EndUser, + app_id: str, + triggered_from: WorkflowNodeExecutionTriggeredFrom, + ) -> OffloadedExecutionRepository: + return OffloadedExecutionRepository( + session_factory=sqlite_session_factory, + tenant_id=tenant_id, + user=user, + app_id=app_id, + triggered_from=triggered_from, + ) + + monkeypatch.setattr( + gateway._workflows, + "_runtime", + replace(runtime_dependencies, node_writer=node_writer), + ) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "Start", + "variables": [ + { + "variable": "payload", + "label": "Payload", + "type": "text-input", + "max_length": 40000, + } + ], + }, + } + ], + "edges": [], + } + ) + response = gateway.run_node( + CONTEXT, "app-1", "start", {"inputs": {"payload": content}}, include_details=True, workflow=None + ) + assert response["status"] == "succeeded" + assert response["outputs_truncated"] is True + assert reads + with sqlite_session_factory() as session: + variable = session.scalar( + select(WorkflowDraftVariable).where( + WorkflowDraftVariable.node_id == "start", + WorkflowDraftVariable.name == "payload", + ) + ) + assert variable is not None + assert variable.get_value().value == content + + +def test_real_start_and_branch_execution_share_injected_variable_and_execution_storage( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = runtime + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "start", + "data": { + "type": "start", + "title": "Start", + "variables": [ + { + "variable": "payload", + "label": "Payload", + "type": "text-input", + "required": True, + "max_length": 100, + } + ], + }, + }, + { + "id": "branch", + "data": { + "type": "if-else", + "title": "Branch", + "cases": [ + { + "case_id": "matches", + "logical_operator": "and", + "conditions": [ + { + "id": "condition", + "varType": "string", + "variable_selector": ["start", "payload"], + "comparison_operator": "is", + "value": "stored value", + } + ], + } + ], + }, + }, + ], + "edges": [], + } + ) + + class ForbiddenDatabase: + @property + def engine(self) -> None: + pytest.fail("execution must not fall back to global db.engine") + + monkeypatch.setattr(workflow_module, "db", ForbiddenDatabase()) + start = gateway.run_node( + CONTEXT, "app-1", "start", {"inputs": {"payload": "stored value"}}, include_details=False, workflow=None + ) + assert start["status"] == "succeeded" + branch = gateway.run_node(CONTEXT, "app-1", "branch", {}, include_details=False, workflow=None) + assert branch["status"] == "succeeded" + assert json.loads(branch["outputs"]) == {"result": True, "selected_case_id": "matches"} + last_run = gateway.last_run(CONTEXT, "app-1", "branch") + assert last_run is not None + assert last_run["id"] == branch["id"] + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("select_all", [False, True], ids=["full-execution", "run-all"]) +@pytest.mark.parametrize("concurrent_change", ["configuration", "delete-node"]) +def test_full_trigger_execution_preserves_revision_through_task_and_worker( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + select_all: bool, + concurrent_change: str, + *, + workflow_variables: WorkflowExecutionVariables, +) -> None: + from core.app.apps import base_app_queue_manager + from core.app.features.rate_limiting.rate_limit import RateLimit + from core.ops.ops_trace_manager import TraceQueueManager + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from services.workflow.execution.adapters.workflow import app_generator as generator_module + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = runtime + gateway._generator = AppGenerateService + original_graph: dict[str, object] = { + "nodes": [ + { + "id": "trigger", + "data": { + "type": "trigger-plugin", + "title": "Subscription A", + "subscription_id": "subscription-A", + }, + } + ], + "edges": [], + } + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps(original_graph) + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + changed = json.loads(draft.graph) + if concurrent_change == "delete-node": + changed["nodes"] = [] + else: + changed["nodes"][0]["data"]["subscription_id"] = "subscription-B" + draft.graph = json.dumps(changed) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "from-A"}}) + + def poller(*, draft_workflow: Workflow, **_kwargs: object) -> Poller: + assert draft_workflow.graph_dict == original_graph + return Poller() + + def select_events(*, draft_workflow: Workflow, **kwargs: object) -> TriggerDebugEvent: + return poller(draft_workflow=draft_workflow, **kwargs).poll() + + monkeypatch.setattr(module, "create_event_poller", poller) + monkeypatch.setattr(module, "select_trigger_debug_events", select_events) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") + limiter = create_autospec(RateLimit, instance=True, spec_set=True) + limiter.generate.side_effect = lambda response, *_args: response + + def guardrails(*, action: Callable[[RateLimit, str], object], **_kwargs: object) -> object: + return action(limiter, "request") + + monkeypatch.setattr(AppGenerateService, "_run_with_guardrails", guardrails) + queued: list[str] = [] + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", queued.append) + + def retrieve_events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str]: + assert all(session.closed for session in sessions) + on_subscribe() + yield from () + + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", retrieve_events) + service = ConsoleWorkflowService( + conversion=create_autospec(WorkflowConversion, instance=True, spec_set=True), + agent_services=WorkflowAgentPublishService, + drafts=create_autospec(WorkflowDraftService, instance=True, spec_set=True), + definitions=create_autospec(WorkflowDefinitions, instance=True, spec_set=True), + lifecycle=create_autospec(WorkflowDefinitionLifecycle, instance=True, spec_set=True), + runtime=gateway, + apps=create_autospec(WorkflowAppLookup, instance=True, spec_set=True), + presence=create_autospec(WorkflowPresence, instance=True, spec_set=True), + access=create_autospec(WorkflowAccess, instance=True, spec_set=True), + ) + with Flask(__name__).app_context(): + response = service.trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=select_all) + assert isinstance(response, Generator) + assert list(response) == [] + assert len(queued) == 1 + params = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert params.workflow_snapshot is not None + assert json.loads(params.workflow_snapshot.graph) == original_graph + + # Leave the real task runner and generator/thread boundary intact. Only external + # transport, tracing and graph execution are replaced with local observers. + observed: list[tuple[dict[str, object], dict[str, object], str | None]] = [] + + class GraphRunner: + def __init__( + self, + *, + workflow: Workflow, + application_generate_entity: WorkflowAppGenerateEntity, + root_node_id: str | None, + **_kwargs: object, + ) -> None: + observed.append((dict(workflow.graph_dict), dict(application_generate_entity.inputs), root_node_id)) + + def run(self) -> None: + pass + + def empty_response(*_args: object, **_kwargs: object) -> Generator[str]: + yield from () + + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", lambda *_args, **_kwargs: None) + trace_manager = create_autospec(TraceQueueManager, instance=True, spec_set=True) + monkeypatch.setattr(generator_module, "TraceQueueManager", lambda **_kwargs: trace_manager) + monkeypatch.setattr(generator_module, "WorkflowAppRunner", GraphRunner) + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", empty_response) + monkeypatch.setattr(task_module, "_publish_streaming_response", lambda response, *_args, **_kwargs: list(response)) + with Flask(__name__).app_context(): + task_module._AppRunner(gateway._sessions, params, variables=workflow_variables).run() + assert observed == [(original_graph, {"payload": "from-A"}, "trigger")] + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("workflow_snapshot.tenant_id", "other-tenant"), + ("workflow_snapshot.app_id", "other-app"), + ("workflow_snapshot.id", "other-workflow"), + ("tenant_id", "other-tenant"), + ("app_id", "other-app"), + ("app_mode", "advanced-chat"), + ("invoke_from", "service-api"), + ], +) +def test_task_rejects_snapshot_outside_its_debugger_owner( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + field: str, + value: str, + *, + workflow_variables: WorkflowExecutionVariables, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from tasks.app_generate.workflow_execute_task import AppExecutionParams, _AppRunner + + gateway, sessions = runtime + app, actor, draft = gateway._draft(CONTEXT, "app-1", "Workflow not found") + params = AppExecutionParams.new( + app_model=app, + workflow=draft, + user=actor, + args={"inputs": {}}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_snapshot=draft, + ) + payload = json.loads(params.model_dump_json()) + if field.startswith("workflow_snapshot."): + payload["workflow_snapshot"][field.split(".")[1]] = value + else: + payload[field] = value + decoded = AppExecutionParams.model_validate_json(json.dumps(payload)) + with pytest.raises(ValueError, match="snapshot does not belong"): + _AppRunner(gateway._sessions, decoded, variables=workflow_variables).run() + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +@pytest.mark.parametrize("entry", ["iteration", "loop"]) +def test_container_debug_prepares_database_records_before_redis( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + mode: AppMode, + entry: Literal["iteration", "loop"], +) -> None: + import threading + + from core.app.apps import base_app_queue_manager + from models.model import App, Conversation, Message + from services.workflow.execution.adapters.chatflow.app_generator import AdvancedChatAppGenerator + from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator + + gateway, sessions = runtime + gateway._generator = AppGenerateService + with sqlite_session_factory.begin() as session: + app = session.get(App, "app-1") + assert app is not None + app.mode = mode + writes: list[str] = [] + + def setex(key: str, *_args: object) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + if mode == AppMode.ADVANCED_CHAT: + with sqlite_session_factory() as session: + assert session.scalar(select(Conversation.id)) is not None + assert session.scalar(select(Message.id)) is not None + writes.append(key) + + def response(*_args: object, **_kwargs: object) -> Generator[str]: + yield from () + + def unexpected_gateway_session() -> NoReturn: + raise AssertionError("Single-node generation must use the repositories' sessions") + + monkeypatch.setattr(gateway, "_sessions", unexpected_gateway_session) + + # Leave actual generator initialization, record creation and queue initialization intact. + monkeypatch.setattr(base_app_queue_manager.redis_client, "setex", setex) + monkeypatch.setattr(threading.Thread, "start", lambda _thread: None) + monkeypatch.setattr(threading.Thread, "join", lambda _thread, **_kwargs: None) + monkeypatch.setattr(WorkflowAppGenerator, "_handle_response", response) + monkeypatch.setattr(AdvancedChatAppGenerator, "_handle_advanced_chat_response", response) + debug_entries = { + "iteration": gateway.iteration, + "loop": gateway.loop, + } + with Flask(__name__).app_context(): + result = debug_entries[entry](CONTEXT, "app-1", "container", {"item": "value"}) + list(result) + assert writes + assert all(session.closed and not session.in_transaction() for session in sessions) + + +@pytest.fixture +def agent_runtime( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> tuple[WorkflowRuntimeGateway, list[RecordingSession]]: + from enums.agent import WorkflowAgentBindingType + from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus, WorkflowAgentNodeBinding + from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig + + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == "draft")) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + {"id": "trigger", "data": {"type": "trigger-plugin"}}, + {"id": "agent-node", "data": {"type": "agent", "version": "2", "agent_node_kind": "dify_agent"}}, + ], + "edges": [], + } + ) + session.add( + Agent( + id="agent-1", + tenant_id="tenant-1", + name="Inline", + scope=AgentScope.WORKFLOW_ONLY, + source=AgentSource.WORKFLOW, + status=AgentStatus.ACTIVE, + ) + ) + for number in (1, 2): + session.add( + AgentConfigSnapshot( + id=f"soul-{number}", + tenant_id="tenant-1", + agent_id="agent-1", + version=number, + config_snapshot=AgentSoulConfig(prompt={"system_prompt": f"Soul {number}"}), + ) + ) + session.add( + WorkflowAgentNodeBinding( + id="binding", + tenant_id="tenant-1", + app_id="app-1", + workflow_id=draft.id, + workflow_version="draft", + node_id="agent-node", + agent_id="agent-1", + binding_type=WorkflowAgentBindingType.INLINE_AGENT, + current_snapshot_id="soul-1", + node_job_config=WorkflowNodeJobConfig(workflow_prompt="Job A"), + ) + ) + + return runtime + + +@pytest.mark.parametrize("change", ["delete", "replace"]) +@pytest.mark.parametrize("select_all", [False, True]) +def test_trigger_pins_agent_job_and_soul_until_worker_finishes( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + change: str, + select_all: bool, +) -> None: + from core.app.features.rate_limiting.rate_limit import RateLimit + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.agent_config_entities import WorkflowNodeJobConfig + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + + class Poller: + def poll(self) -> TriggerDebugEvent: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + if change == "delete": + session.delete(binding) + draft = session.get(Workflow, binding.workflow_id) + assert draft is not None + draft.graph = '{"nodes": [], "edges": []}' + else: + binding.current_snapshot_id = "soul-2" + binding.node_job_config = WorkflowNodeJobConfig(workflow_prompt="Job B") + session.flush() + assert ( + WorkflowAgentRetirementRepository.archive_unowned( + session=session, tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1" + ) + == [] + ) + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {"payload": "A"}}) + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + monkeypatch.setattr(module, "select_trigger_debug_events", lambda **_kwargs: Poller().poll()) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE="streams") + limiter = create_autospec(RateLimit, instance=True, spec_set=True) + limiter.generate.side_effect = lambda response, *_args: response + monkeypatch.setattr( + AppGenerateService, "_run_with_guardrails", lambda *, action, **_kwargs: action(limiter, "request") + ) + queued: list[str] = [] + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", queued.append) + + def events(*_args: object, on_subscribe: Callable[[], None]) -> Generator[str]: + on_subscribe() + yield from () + + monkeypatch.setattr(WorkflowEventStream, "retrieve_events", events) + with Flask(__name__).app_context(): + event_data = gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=select_all) + assert event_data is not None + assert event_data.workflow.execution_id is not None + list( + gateway.generate( + CONTEXT, "app-1", event_data.workflow_args, root_node_id="trigger", workflow=event_data.workflow + ) + ) + params = task_module.AppExecutionParams.model_validate_json(queued[0]) + assert params.workflow_run_id == event_data.workflow.execution_id + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + def run_app(_runner, *, workflow: Workflow, **_kwargs: object) -> Generator[str]: + assert workflow.graph_dict["nodes"][1]["id"] == "agent-node" + resolved = _runner._runtime.agent_bindings.resolve( + tenant_id="tenant-1", + app_id="app-1", + workflow_id=workflow.id, + node_id="agent-node", + workflow_run_id=params.workflow_run_id, + ) + assert resolved.binding.node_job_config_dict["workflow_prompt"] == "Job A" + assert resolved.snapshot.id == "soul-1" + assert resolved.snapshot.config_snapshot_dict["prompt"]["system_prompt"] == "Soul 1" + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, params.workflow_run_id) + assert run is not None + run.status = WorkflowExecutionStatus.SUCCEEDED + yield from () + + monkeypatch.setattr(task_module._AppRunner, "_run_app", run_app) + monkeypatch.setattr(task_module, "_publish_streaming_response", lambda response, *_args, **_kwargs: list(response)) + with Flask(__name__).app_context(): + task_module._AppRunner(gateway._sessions, params, variables=gateway._variables).run() + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == (AgentStatus.ARCHIVED if change == "delete" else AgentStatus.ACTIVE) + assert all(session.closed and not session.in_transaction() for session in sessions) + + +@pytest.mark.parametrize("outcome", ["empty", "poll-error", "prepare-error"]) +def test_trigger_cancels_reserved_agent_references_without_starting_execution( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + outcome: str, +) -> None: + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + class Poller: + def poll(self) -> TriggerDebugEvent | None: + assert sessions + assert all(session.closed for session in sessions) + with sqlite_session_factory.begin() as session: + assert session.scalar(select(WorkflowRun.id)) is not None + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + if outcome == "poll-error": + raise ValueError("poll failed") + if outcome == "empty": + return None + return TriggerDebugEvent(node_id="trigger", workflow_args={"inputs": {}}) + + def denied(**_kwargs: object) -> None: + raise ValueError("prepare failed") + + monkeypatch.setattr(module, "create_event_poller", lambda **_kwargs: Poller()) + monkeypatch.setattr(AppGenerateService, "_run_with_guardrails", denied) + if outcome == "empty": + assert gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) is None + elif outcome == "poll-error": + with pytest.raises(ValueError, match="poll failed"): + gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) + else: + event_data = gateway.poll_trigger(CONTEXT, "app-1", ["trigger"], single_node=False, select_all=False) + assert event_data is not None + with Flask(__name__).app_context(), pytest.raises(ValueError, match="prepare failed"): + gateway.generate(CONTEXT, "app-1", {}, root_node_id="trigger", workflow=event_data.workflow) + with sqlite_session_factory() as session: + assert session.scalar(select(WorkflowRun.id)) is None + agent = session.get(Agent, "agent-1") + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + + +def test_paused_trigger_keeps_agent_references_until_resumed_stream_finishes( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + from graphon.entities import WorkflowStartReason + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate.workflow_execute_task import _publish_streaming_response + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + + class Topic: + def publish(self, _payload: bytes) -> None: + pass + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + for event_name, status in [ + ("workflow_paused", WorkflowExecutionStatus.PAUSED), + ("workflow_finished", WorkflowExecutionStatus.SUCCEEDED), + ]: + _publish_streaming_response( + (item for item in [{"event": event_name, "data": {"status": status.value}}]), + snapshot.execution_id, + AppMode.WORKFLOW, + snapshot.id, + {}, + WorkflowStartReason.RESUMPTION, + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + agent_snapshot=True, + ) + with sqlite_session_factory() as session: + agent = session.get(Agent, "agent-1") + run = session.get(WorkflowRun, snapshot.execution_id) + assert agent is not None + assert run is not None + assert run.status == status + assert agent.status == ( + AgentStatus.ACTIVE if status == WorkflowExecutionStatus.PAUSED else AgentStatus.ARCHIVED + ) + + +@pytest.mark.parametrize("channel_type", ["streams", "pubsub", "sharded"]) +@pytest.mark.parametrize( + "outcome", + [ + "broker-error", + "subscribe-error", + "unopened", + "response-unopened", + "close-before-subscribe", + "disconnect-after-enqueue", + "error-after-enqueue", + "complete", + ], +) +def test_reserved_trigger_stream_owns_cleanup_until_enqueue( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + channel_type: str, + outcome: str, +) -> None: + from core.app.features.rate_limiting.rate_limit import RateLimit + from graphon.enums import WorkflowExecutionStatus + from libs.helper import compact_generate_response + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate import workflow_execute_task as task_module + + gateway, sessions = agent_runtime + gateway._generator = AppGenerateService + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + session.flush() + assert ( + WorkflowAgentRetirementRepository.archive_unowned( + session=session, tenant_id="tenant-1", agent_ids=["agent-1"], account_id="account-1" + ) + == [] + ) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", lambda **_kwargs: None) + apply_config_overrides(monkeypatch, PUBSUB_REDIS_CHANNEL_TYPE=channel_type) + + def unexpected_timer(*_args: object, **_kwargs: object) -> None: + pytest.fail("a reservation must not enqueue after the stream releases its references") + + monkeypatch.setattr("services.app_generate_service.threading.Timer", unexpected_timer) + limiter = RateLimit("app-1", -1) + monkeypatch.setattr( + AppGenerateService, + "_run_with_guardrails", + lambda *, action, **_kwargs: action(limiter, RateLimit._UNLIMITED_REQUEST_ID), + ) + queued: list[str] = [] + + def enqueue(payload: str) -> None: + assert all(session.closed for session in sessions) + if outcome == "broker-error": + raise RuntimeError("broker unavailable") + queued.append(payload) + + monkeypatch.setattr(task_module.workflow_based_app_execution_task, "delay", enqueue) + + class Subscription: + def __enter__(self) -> Self: + if outcome == "subscribe-error": + raise RuntimeError("subscription unavailable") + return self + + def __exit__(self, *_args: object) -> None: + pass + + def receive(self, **_kwargs: object) -> bytes: + if outcome == "error-after-enqueue": + raise RuntimeError("connection lost") + if outcome == "complete": + return b'{"event": "workflow_finished"}' + return b'{"event": "workflow_started"}' + + class Topic: + def as_subscriber(self) -> Self: + return self + + def subscribe(self) -> Subscription: + return Subscription() + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + with Flask(__name__).test_request_context(): + stream = gateway.generate(CONTEXT, "app-1", {}, root_node_id="trigger", workflow=snapshot) + assert isinstance(stream, Generator) + assert not queued + if outcome == "response-unopened": + response = compact_generate_response(stream) + response.close() + elif outcome == "unopened": + stream.close() + elif outcome == "close-before-subscribe": + assert next(stream) == "event: ping\n\n" + stream.close() + elif outcome == "disconnect-after-enqueue": + next(stream) # Initial ping precedes subscribing and dispatch. + next(stream) + stream.close() + elif outcome == "complete": + list(stream) + else: + with pytest.raises(RuntimeError, match="unavailable|connection lost"): + list(stream) + stream.close() # Repeated closure must be harmless. + + enqueued = outcome in {"disconnect-after-enqueue", "error-after-enqueue", "complete"} + assert len(queued) == int(enqueued) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert agent is not None + if enqueued: + assert run is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert agent.status == AgentStatus.ACTIVE + else: + assert run is None + assert agent.status == AgentStatus.ARCHIVED + assert all(session.closed for session in sessions) + + +@pytest.mark.parametrize("other_execution", [None, "running", "paused"]) +def test_global_timeout_retires_paused_trigger_agent_after_last_execution( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + other_execution: str | None, +) -> None: + from datetime import timedelta + from types import SimpleNamespace + + from enums.human_input import HumanInputFormKind, HumanInputFormStatus + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.agent import ( + Agent, + AgentHomeSnapshot, + AgentStatus, + AgentWorkingResourceStatus, + WorkflowAgentNodeBinding, + ) + from models.human_input import HumanInputForm + from models.workflow import WorkflowRun + from tasks import human_input_timeout_tasks as timeout_module + + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + if other_execution is not None: + _, other = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + with sqlite_session_factory.begin() as session: + other_run = session.get(WorkflowRun, other.execution_id) + assert other_run is not None + other_run.status = WorkflowExecutionStatus(other_execution) + now = naive_utc_now() + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert run is not None + assert binding is not None + run.status = WorkflowExecutionStatus.PAUSED + session.delete(binding) + session.add( + AgentHomeSnapshot( + id="home-1", + tenant_id="tenant-1", + agent_id="agent-1", + snapshot_ref="home-ref", + status=AgentWorkingResourceStatus.ACTIVE, + ) + ) + session.add( + HumanInputForm( + id="timeout-form", + tenant_id="tenant-1", + app_id="app-1", + workflow_run_id=run.id, + node_id="agent-node", + form_kind=HumanInputFormKind.RUNTIME, + form_definition='{"form_content": "", "rendered_content": ""}', + rendered_content="", + status=HumanInputFormStatus.WAITING, + created_at=now - timedelta(hours=2), + expiration_time=now + timedelta(hours=1), + ) + ) + assert session.scalar(select(WorkflowAgentNodeBinding.agent_id)) == "agent-1" + monkeypatch.setattr(timeout_module, "db", SimpleNamespace(engine=sqlite_session_factory.kw["bind"])) + apply_config_overrides(monkeypatch, HUMAN_INPUT_GLOBAL_TIMEOUT_SECONDS=3600) + collector = Mock() + monkeypatch.setattr("tasks.collect_agent_resources_task.collect_agent_resources.delay", collector) + + timeout_module.check_and_handle_human_input_timeouts() + + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + home = session.get(AgentHomeSnapshot, "home-1") + form = session.get(HumanInputForm, "timeout-form") + assert run is not None + assert agent is not None + assert home is not None + assert form is not None + assert run.status == WorkflowExecutionStatus.STOPPED + assert form.status == HumanInputFormStatus.EXPIRED + assert run.finished_at is not None + if other_execution is None: + assert agent.status == AgentStatus.ARCHIVED + assert home.status == AgentWorkingResourceStatus.RETIRED + collector.assert_called_once() + assert collector.call_args.kwargs["purge_agent_ids"] == ["agent-1"] + assert collector.call_args.kwargs["home_snapshot_ids"] == ["home-1"] + else: + assert agent.status == AgentStatus.ACTIVE + assert home.status == AgentWorkingResourceStatus.ACTIVE + collector.assert_not_called() + assert all(session.closed for session in sessions) + + +def test_running_lease_renews_and_fences_dead_workers( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + from datetime import timedelta + + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowRun + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + from repositories.workflow.execution_write_repository import save_workflow_run + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + now = naive_utc_now() + timeout = timedelta(seconds=dify_config.WORKFLOW_DEBUG_RESERVATION_TIMEOUT) + repository = WorkflowDebugReservationRepository(sqlite_session_factory) + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + WorkflowDebugReservationRepository.claim(session, run, now) + # A run can outlive its original startup deadline while its worker renews. + for _ in range(4): + now += timeout / 2 + assert repository.renew( + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + now=now, + ) + assert repository.pending_batch(now, limit=100) == [] + assert repository.expire(snapshot.execution_id, now) is None + with sqlite_session_factory() as session: + stale_start = session.get(WorkflowRun, snapshot.execution_id) + assert stale_start is not None + session.expunge(stale_start) + # Simulate process death: no more heartbeats, even if no Celery terminal task arrives. + now += timedelta(days=365) + assert [item.execution_id for item in repository.pending_batch(now, limit=100)] == [snapshot.execution_id] + assert ( + repository.renew( + tenant_id=snapshot.tenant_id, + app_id=snapshot.app_id, + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + now=now, + ) + is False + ) + assert repository.expire(snapshot.execution_id, now) is not None + with sqlite_session_factory.begin() as session: + assert save_workflow_run(session, stale_start) is False + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + reservation = debug_lease(session, snapshot.execution_id) + assert run is not None + assert run.status.value == "failed" + assert run.error == "Trigger debug worker lease expired" + # Cleanup retry evidence survives the terminal transition. + assert reservation is not None + + +def test_paused_debug_lease_is_protected_and_refreshed_atomically_on_resume( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + from datetime import timedelta + + from graphon.enums import WorkflowExecutionStatus + from libs.datetime_utils import naive_utc_now + from models.workflow import WorkflowPause, WorkflowRun + from repositories.sqlalchemy_api_workflow_run_repository import DifyAPISQLAlchemyWorkflowRunRepository + from repositories.workflow.debug_reservation_repository import WorkflowDebugReservationRepository + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + now = naive_utc_now() + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + lease = debug_lease(session, snapshot.execution_id) + assert run is not None + assert lease is not None + WorkflowDebugReservationRepository.claim(session, run, now) + run.status = WorkflowExecutionStatus.PAUSED + set_debug_deadline(session, snapshot.execution_id, now - timedelta(days=365)) + session.add(WorkflowPause(workflow_id=run.workflow_id, workflow_run_id=run.id, state_object_key="pause")) + reservations = WorkflowDebugReservationRepository(sqlite_session_factory) + assert reservations.pending_batch(now + timedelta(days=365), limit=100) == [] + assert reservations.expire(snapshot.execution_id, now) is None + runs = DifyAPISQLAlchemyWorkflowRunRepository(sqlite_session_factory) + pause = runs.get_workflow_pause(snapshot.execution_id) + assert pause is not None + runs.resume_workflow_pause(snapshot.execution_id, pause) + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + lease = debug_lease(session, snapshot.execution_id) + assert run is not None + assert lease is not None + assert run.status == WorkflowExecutionStatus.RUNNING + assert lease.expires_at is not None + assert lease.expires_at > now + assert reservations.expire(snapshot.execution_id, now) is None + + +@pytest.mark.parametrize("failed_row", ["reservation", "reference"]) +def test_reservation_and_execution_owners_roll_back_together( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + failed_row: str, +) -> None: + from models.workflow import WorkflowRun + + gateway, _ = agent_runtime + from models.agent import WorkflowAgentNodeBinding + + row_type = WorkflowRun if failed_row == "reservation" else WorkflowAgentNodeBinding + + def fail(session: Session, _context: object, _instances: object) -> None: + if any(isinstance(row, row_type) for row in session.new): + raise RuntimeError("reservation persistence failed") + + event.listen(gateway._reservations._sessions, "before_flush", fail) + try: + with pytest.raises(RuntimeError, match="reservation persistence failed"): + gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + finally: + event.remove(gateway._reservations._sessions, "before_flush", fail) + with sqlite_session_factory() as session: + assert session.scalar(select(WorkflowRun)) is None + assert ( + session.scalar( + select(WorkflowAgentNodeBinding).where(WorkflowAgentNodeBinding.workflow_version == "execution") + ) + is None + ) + + +@pytest.mark.parametrize("cleanup_fails", [True, False]) +def test_terminal_notification_survives_cleanup_dispatch_failure( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + cleanup_fails: bool, +) -> None: + from graphon.entities import WorkflowStartReason + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from services.agent.retirement_service import WorkflowAgentRetirementService + from services.workflow.execution.adapters.response_stream import WorkflowEventStream + from tasks.app_generate.workflow_execute_task import _publish_streaming_response + + gateway, _ = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + received: list[dict[str, object]] = [] + attempted: list[set[str]] = [] + + class Topic: + def publish(self, payload: bytes) -> None: + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.status == WorkflowExecutionStatus.SUCCEEDED + received.append(json.loads(payload)) + + def dispatch(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + assert received == [{"event": "workflow_finished", "data": {"status": "succeeded"}}] + attempted.append(set(purge_agent_ids)) + if cleanup_fails and len(attempted) == 1: + raise RuntimeError("resource broker unavailable") + + monkeypatch.setattr(WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", dispatch) + _publish_streaming_response( + (item for item in [{"event": "workflow_finished", "data": {"status": "succeeded"}}]), + snapshot.execution_id, + AppMode.WORKFLOW, + snapshot.id, + {}, + WorkflowStartReason.RESUMPTION, + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + agent_snapshot=True, + ) + assert len(received) == 1 + assert attempted == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == WorkflowExecutionStatus.SUCCEEDED + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails + if cleanup_fails: + # The real entry point must recover candidates without callers retaining Agent IDs. + WorkflowAgentRetirementService.finish_execution( + sessions=gateway._sessions, + tenant_id="tenant-1", + app_id="app-1", + workflow_id=snapshot.id, + execution_id=snapshot.execution_id, + account_id=None, + ) + assert attempted == [{"agent-1"}, {"agent-1"}] + with sqlite_session_factory() as session: + assert debug_lease(session, snapshot.execution_id) is None + assert len(received) == 1 + + +@pytest.mark.parametrize("outcome", ["succeeded", "failed", "stopped", "partial-succeeded"]) +@pytest.mark.parametrize("cleanup_fails", [False, True]) +def test_completed_debug_redelivery_replays_persisted_outcome_without_execution_or_cleanup( + agent_runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + outcome: str, + cleanup_fails: bool, +) -> None: + from core.app.entities.app_invoke_entities import InvokeFrom + from core.app.entities.task_entities import WorkflowFinishStreamResponse + from graphon.enums import WorkflowExecutionStatus + from models.agent import Agent, AgentStatus, WorkflowAgentNodeBinding + from models.workflow import WorkflowRun + from tasks.app_generate import workflow_execute_task as task_module + + status = WorkflowExecutionStatus(outcome) + gateway, sessions = agent_runtime + _, snapshot = gateway._reservations.reserve_trigger_debug(CONTEXT, "app-1") + assert snapshot.execution_id is not None + app, actor, workflow = gateway._definitions.debug_context(CONTEXT, "app-1", workflow_id=None, snapshot=snapshot) + params = task_module.AppExecutionParams.new( + app_model=app, + workflow=workflow, + user=actor, + args={}, + invoke_from=InvokeFrom.DEBUGGER, + workflow_run_id=snapshot.execution_id, + workflow_snapshot=snapshot, + ) + with sqlite_session_factory.begin() as session: + binding = session.get(WorkflowAgentNodeBinding, "binding") + assert binding is not None + session.delete(binding) + + engine_calls: list[str] = [] + cleanup_calls: list[set[str]] = [] + received: list[bytes] = [] + execution_error = None if status == WorkflowExecutionStatus.SUCCEEDED else "Actual engine outcome" + + def execute(_runner: task_module._AppRunner, **_kwargs: object) -> Generator[dict[str, object]]: + engine_calls.append(params.workflow_run_id) + yield {"event": "workflow_started"} + with sqlite_session_factory.begin() as session: + run = session.get(WorkflowRun, params.workflow_run_id) + assert run is not None + run.outputs = json.dumps({"answer": "stored output"}) + run.error = execution_error + run.elapsed_time = 1.5 + run.total_tokens = 42 + run.total_steps = 3 + yield {"event": "workflow_finished", "data": {"status": status.value}} + + def collect(*, purge_agent_ids: Iterable[str], **_kwargs: object) -> None: + cleanup_calls.append(set(purge_agent_ids)) + if cleanup_fails: + raise RuntimeError("Cleanup broker unavailable") + + class Topic: + def publish(self, payload: bytes) -> None: + assert all(session.closed and not session.in_transaction() for session in sessions) + received.append(payload) + + monkeypatch.setattr(task_module._AppRunner, "_run_app", execute) + monkeypatch.setattr(task_module.WorkflowEventStream, "get_response_topic", lambda *_args: Topic()) + monkeypatch.setattr("services.agent.retirement_service.enqueue_agent_resource_collection", collect) + with Flask(__name__).app_context(): + assert task_module._AppRunner(gateway._sessions, params, variables=gateway._variables).run() is None + assert [json.loads(payload)["event"] for payload in received] == ["workflow_started", "workflow_finished"] + assert engine_calls == [snapshot.execution_id] + assert cleanup_calls == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + agent = session.get(Agent, "agent-1") + assert run is not None + assert run.status == status + assert agent is not None + assert agent.status == AgentStatus.ARCHIVED + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails + persisted = run.to_dict() + + received.clear() + # The delivery boundary deserializes a fresh copy of the original task payload. + for _ in range(2): + redelivery = task_module.AppExecutionParams.model_validate_json(params.model_dump_json()) + assert task_module._AppRunner(gateway._sessions, redelivery, variables=gateway._variables).run() is None + assert [json.loads(payload)["event"] for payload in received] == ["workflow_finished", "workflow_finished"] + for payload in received: + replay = WorkflowFinishStreamResponse.model_validate_json(payload) + assert replay.workflow_run_id == snapshot.execution_id + assert replay.data.status == status + assert replay.data.error == execution_error + assert replay.data.outputs == {"answer": "stored output"} + assert replay.data.elapsed_time == 1.5 + assert replay.data.total_steps == 3 + assert replay.data.total_tokens == 42 + + # Blocking redeliveries have no stream to replay and also exit without work. + blocking = params.model_copy(update={"streaming": False}) + assert task_module._AppRunner(gateway._sessions, blocking, variables=gateway._variables).run() is None + assert len(received) == 2 + assert engine_calls == [snapshot.execution_id] + assert cleanup_calls == [{"agent-1"}] + with sqlite_session_factory() as session: + run = session.get(WorkflowRun, snapshot.execution_id) + assert run is not None + assert run.to_dict() == persisted + assert (debug_lease(session, snapshot.execution_id) is not None) == cleanup_fails diff --git a/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py b/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py index 38434e10e3c593..80da7a9bb82496 100644 --- a/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py +++ b/api/tests/unit_tests/services/workflow/test_runtime_variable_files.py @@ -16,6 +16,7 @@ from models.dataset import Pipeline from models.snippet import CustomizedSnippet from models.workflow import Workflow, WorkflowDraftVariable, WorkflowKind, WorkflowType +from repositories.workflow.definition_repository import workflow_snapshot from services.workflow.variable_service import WorkflowVariableService from tests.helpers.unit.model_factories import make_account, make_app, make_upload_file, make_workflow @@ -106,7 +107,7 @@ def test_load_file_variables_restores_canonical_metadata( _persist_file_variable( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) assert len(loaded) == 1 assert loaded[0].selector == ["node", "file"] files = loaded[0].value if value_type == SegmentType.ARRAY_FILE else [loaded[0].value] @@ -135,7 +136,7 @@ def test_load_file_variables_rejects_another_tenants_upload( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) with pytest.raises(ValueError, match="Invalid upload file"): - variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) @pytest.mark.parametrize("value_type", [SegmentType.FILE, SegmentType.ARRAY_FILE]) @@ -156,7 +157,7 @@ def test_load_external_file_does_not_fetch_it_again( _persist_file_variable( sqlite_session_factory, workflow, value_type, [mapping] if value_type == SegmentType.ARRAY_FILE else mapping ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) file = loaded[0].value[0] if value_type == SegmentType.ARRAY_FILE else loaded[0].value assert (file.remote_url, file.filename, file.size) == (mapping["url"], "document.txt", 12) @@ -166,7 +167,7 @@ def test_load_empty_file_array( ) -> None: variables, workflow = runtime_scope _persist_file_variable(sqlite_session_factory, workflow, SegmentType.ARRAY_FILE, []) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["node", "file"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["node", "file"]]) assert loaded[0].value == [] @@ -192,5 +193,5 @@ def test_start_node_files_round_trip_through_injected_gateway( ] }, ) - loaded = variables.workflow_loader(workflow, "account-1").load_variables([["sys", "files"]]) + loaded = variables.workflow_loader(workflow_snapshot(workflow), "account-1").load_variables([["sys", "files"]]) assert [file.filename for file in loaded[0].value] == ["start.txt"] diff --git a/api/tests/unit_tests/services/workflow/test_variable_service.py b/api/tests/unit_tests/services/workflow/test_variable_service.py index 87984e87a355f3..fddb5a698f086b 100644 --- a/api/tests/unit_tests/services/workflow/test_variable_service.py +++ b/api/tests/unit_tests/services/workflow/test_variable_service.py @@ -12,18 +12,16 @@ from extensions.application_services.workflow_variables import build_workflow_variable_service from extensions.ext_storage import storage from graphon.nodes import BuiltinNodeTypes -from graphon.variable_loader import VariableLoader from graphon.variables import SegmentType, StringVariable from machinery.context import RequestContext from models import UploadFile from models.account import TenantAccountJoin, TenantAccountRole from models.workflow import WorkflowDraftVariable, WorkflowDraftVariableFile from repositories.factory import DifyAPIRepositoryFactory -from repositories.workflow.definition_repository import workflow_from_snapshot +from repositories.workflow.definition_repository import workflow_snapshot from repositories.workflow.draft_variable_repository import WorkflowDraftVariableRepository from services.app.generation.input_adapter import AppInputAdapter from services.file_service import FileService -from services.workflow.contracts import WorkflowSnapshot from services.workflow.variable_file_gateway import WorkflowVariableFileGateway from services.workflow.variable_service import WorkflowVariableService from tasks.workflow_draft_var_tasks import cleanup_draft_variable_files_task, recover_draft_variable_file_cleanup_task @@ -71,17 +69,13 @@ def delete(key: str) -> None: monkeypatch.setattr(storage, "load", download) monkeypatch.setattr(storage, "delete", delete) variables = build_workflow_variable_service(database_client=factory) - - def load_draft_variables(workflow: WorkflowSnapshot, user_id: str) -> VariableLoader: - return variables.workflow_loader(workflow_from_snapshot(workflow), user_id) - generator = AppInputAdapter( - draft_variable_loader=load_draft_variables, draft_variable_saver=variables.saver_factory + draft_variable_loader=variables.workflow_loader, draft_variable_saver=variables.saver_factory ) account = make_account() workflow = make_workflow() workflow.conversation_variables = [StringVariable(name="greeting", value="hello")] - loader = variables.workflow_loader(workflow, account.id) + loader = variables.workflow_loader(workflow_snapshot(workflow), account.id) assert loader.load_variables([["conversation", "greeting"]])[0].value == "hello" saver_factory = generator._get_draft_var_saver_factory(InvokeFrom.DEBUGGER, account, tenant_id=workflow.tenant_id) @@ -341,7 +335,7 @@ def unexpected_global_read(_filename: str) -> bytes: assert len(contents) == 2 loader = ( - variables.workflow_loader(workflow, account.id) + variables.workflow_loader(workflow_snapshot(workflow), account.id) if loader_entry == "workflow" else variables.loader(RequestContext("request", None, account.id, workflow.tenant_id), workflow.app_id, []) ) diff --git a/api/tests/unit_tests/services/workflow/test_workflow_converter.py b/api/tests/unit_tests/services/workflow/test_workflow_converter.py index b95f0fd43e5a15..dc24dc2c74b0c3 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_converter.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_converter.py @@ -1,13 +1,9 @@ from __future__ import annotations import json -from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock, call import pytest -from sqlalchemy import event, select -from sqlalchemy.orm import Session from core.app.app_config.entities import ( AdvancedChatMessageEntity, @@ -19,11 +15,10 @@ ModelConfigEntity, PromptTemplateEntity, ) -from core.helper import encrypter from core.prompt.utils.prompt_template_parser import PromptTemplateParser -from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint -from models.model import Account, App, AppMode, AppModelConfig, IconType -from models.workflow import Workflow, WorkflowType +from models.api_based_extension import APIBasedExtensionPoint +from models.model import App, AppMode +from models.workflow_conversion import ConversionExtension from services.workflow import workflow_converter as converter_module from services.workflow.workflow_converter import WorkflowConverter @@ -59,24 +54,6 @@ def _app_model(**kwargs: Any) -> App: return App(**defaults) -def _account(**kwargs: Any) -> Account: - account_id = kwargs.pop("id", "account-1") - account = Account( - name=kwargs.pop("name", "Converter user"), - email=kwargs.pop("email", "user@example.com"), - **kwargs, - ) - account.id = account_id - return account - - -def _app_model_config(**kwargs: Any) -> AppModelConfig: - config_id = kwargs.pop("id", "config-1") - config = AppModelConfig(app_id=kwargs.pop("app_id", "app-1"), **kwargs) - config.id = config_id - return config - - def _build_start_graph() -> dict[str, Any]: return { "nodes": [ @@ -112,22 +89,17 @@ def test__convert_to_start_node(default_variables: list[VariableEntity]) -> None assert result["data"]["variables"][0]["variable"] == "text_input" -def test__convert_to_http_request_node_for_chatbot( - default_variables: list[VariableEntity], unbound_session: Session -) -> None: +def test__convert_to_http_request_node_for_chatbot(default_variables: list[VariableEntity]) -> None: app_model = _app_model(id="app_id", tenant_id="tenant_id", mode=AppMode.CHAT) - extension = APIBasedExtension( - tenant_id="tenant_id", + extension = ConversionExtension( + id="api_based_extension_id", name="api-1", - api_key="encrypted_api_key", + api_key="api_key", api_endpoint="https://dify.ai", ) - extension.id = "api_based_extension_id" workflow_converter = WorkflowConverter() - workflow_converter._get_api_based_extension = MagicMock(return_value=extension) - encrypter.decrypt_token = MagicMock(return_value="api_key") external_data_variables = [ ExternalDataVariableEntity( @@ -138,10 +110,11 @@ def test__convert_to_http_request_node_for_chatbot( ] nodes, mapping = workflow_converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=default_variables, external_data_variables=external_data_variables, - session=unbound_session, + extensions={extension.id: extension}, ) assert len(nodes) == 2 @@ -154,22 +127,17 @@ def test__convert_to_http_request_node_for_chatbot( assert mapping == {"external_variable": "code_1"} -def test__convert_to_http_request_node_for_workflow_app( - default_variables: list[VariableEntity], unbound_session: Session -) -> None: +def test__convert_to_http_request_node_for_workflow_app(default_variables: list[VariableEntity]) -> None: app_model = _app_model(id="app_id", tenant_id="tenant_id", mode=AppMode.WORKFLOW) - extension = APIBasedExtension( - tenant_id="tenant_id", + extension = ConversionExtension( + id="api_based_extension_id", name="api-1", - api_key="encrypted_api_key", + api_key="api_key", api_endpoint="https://dify.ai", ) - extension.id = "api_based_extension_id" workflow_converter = WorkflowConverter() - workflow_converter._get_api_based_extension = MagicMock(return_value=extension) - encrypter.decrypt_token = MagicMock(return_value="api_key") external_data_variables = [ ExternalDataVariableEntity( @@ -180,10 +148,11 @@ def test__convert_to_http_request_node_for_workflow_app( ] nodes, _ = workflow_converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=default_variables, external_data_variables=external_data_variables, - session=unbound_session, + extensions={extension.id: extension}, ) body = json.loads(nodes[0]["data"]["body"]["data"]) @@ -377,328 +346,8 @@ def test__convert_to_answer_node() -> None: assert node["data"]["type"] == BuiltinNodeTypes.ANSWER -def test_convert_to_workflow_should_raise_when_app_model_config_is_missing( - converter: WorkflowConverter, unbound_session: Session -) -> None: - app_model = _app_model(app_model_config_id=None) - - with pytest.raises(ValueError, match="App model config is required"): - converter.convert_to_workflow( - app_model=app_model, - account=_account(id="account-1"), - name="new-app", - icon_type="emoji", - icon="robot", - icon_background="#fff", - session=unbound_session, - ) - - assert not unbound_session.in_transaction() - - -@pytest.mark.parametrize( - ("source_mode", "expected_mode"), - [ - (AppMode.CHAT, AppMode.ADVANCED_CHAT), - (AppMode.COMPLETION, AppMode.WORKFLOW), - ], -) -def test_convert_to_workflow_should_create_new_app_with_fallback_fields( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - source_mode: AppMode, - expected_mode: AppMode, - sqlite_session: Session, -) -> None: - app_model_config = AppModelConfig(app_id="source-app") - app_model_config.id = "config-1" - sqlite_session.add(app_model_config) - sqlite_session.flush() - workflow = Workflow( - tenant_id="tenant-1", - app_id="source-app", - type=WorkflowType.WORKFLOW, - version=Workflow.VERSION_DRAFT, - graph="{}", - _features="{}", - created_by="account-1", - _environment_variables=[], - _conversation_variables=[], - ) - monkeypatch.setattr(converter, "convert_app_model_config_to_workflow", MagicMock(return_value=workflow)) - phase_events: list[str] = [] - event.listen(sqlite_session, "after_commit", lambda _session: phase_events.append("commit")) - - send_mock = MagicMock(side_effect=lambda *_args, **_kwargs: phase_events.append("signal")) - monkeypatch.setattr(converter_module.app_was_created, "send", send_mock) - - account = _account(id="account-1") - app_model = _app_model( - tenant_id="tenant-1", - id="source-app", - name="Source App", - mode=source_mode, - icon_type=IconType.EMOJI, - icon="sparkles", - icon_background="#123456", - enable_site=True, - enable_api=True, - api_rpm=10, - api_rph=100, - is_public=False, - app_model_config_id="config-1", - ) - - new_app = converter.convert_to_workflow( - app_model=app_model, - account=account, - name="", - icon_type="", - icon="", - icon_background="", - session=sqlite_session, - ) - - assert new_app.name == "Source App(workflow)" - assert new_app.mode == expected_mode - assert new_app.icon_type == IconType.EMOJI - assert new_app.icon == "sparkles" - assert new_app.icon_background == "#123456" - assert new_app.created_by == "account-1" - assert workflow.app_id == new_app.id - assert sqlite_session.get(App, new_app.id) is new_app - assert phase_events == ["commit", "signal", "commit"] - send_mock.assert_called_once_with(new_app, account=account, session=sqlite_session) - - -def test_convert_app_model_config_to_workflow_should_build_advanced_chat_graph_and_features( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.CHAT) - app_config = SimpleNamespace( - variables=[SimpleNamespace(variable="name")], - external_data_variables=[SimpleNamespace(variable="ext")], - dataset=SimpleNamespace(id="dataset"), - model=SimpleNamespace(), - prompt_template=SimpleNamespace(), - additional_features=SimpleNamespace(file_upload=SimpleNamespace()), - app_model_config_dict={ - "opening_statement": "hello", - "suggested_questions": ["q1"], - "suggested_questions_after_answer": True, - "speech_to_text": True, - "text_to_speech": {"enabled": True}, - "file_upload": {"enabled": True}, - "sensitive_word_avoidance": {"enabled": True}, - "retriever_resource": {"enabled": True}, - }, - ) - - monkeypatch.setattr(converter, "_get_new_app_mode", MagicMock(return_value=AppMode.ADVANCED_CHAT)) - monkeypatch.setattr(converter, "_convert_to_app_config", MagicMock(return_value=app_config)) - monkeypatch.setattr( - converter, - "_convert_to_start_node", - MagicMock( - return_value={"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START, "variables": []}} - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_http_request_node", - MagicMock( - return_value=( - [{"id": "http", "position": None, "data": {"type": BuiltinNodeTypes.HTTP_REQUEST}}], - {"ext": "code_1"}, - ) - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_knowledge_retrieval_node", - MagicMock( - return_value={"id": "knowledge", "position": None, "data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}} - ), - ) - monkeypatch.setattr( - converter, - "_convert_to_llm_node", - MagicMock(return_value={"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}}), - ) - monkeypatch.setattr( - converter, - "_convert_to_answer_node", - MagicMock(return_value={"id": "answer", "position": None, "data": {"type": BuiltinNodeTypes.ANSWER}}), - ) - workflow = converter.convert_app_model_config_to_workflow( - app_model=app_model, - app_model_config=_app_model_config(id="cfg"), - account_id="account-1", - session=sqlite_session, - ) - - graph = json.loads(workflow.graph) - node_ids = [node["id"] for node in graph["nodes"]] - assert node_ids == ["start", "http", "knowledge", "llm", "answer"] - - features = json.loads(workflow.features) - assert "opening_statement" in features - assert "retriever_resource" in features - assert sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) is workflow - - -def test_convert_app_model_config_to_workflow_should_build_workflow_mode_with_end_node( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.COMPLETION) - app_config = SimpleNamespace( - variables=[SimpleNamespace(variable="name")], - external_data_variables=[], - dataset=SimpleNamespace(id="dataset"), - model=SimpleNamespace(), - prompt_template=SimpleNamespace(), - additional_features=None, - app_model_config_dict={ - "text_to_speech": {"enabled": False}, - "file_upload": {"enabled": False}, - "sensitive_word_avoidance": {"enabled": False}, - }, - ) - - monkeypatch.setattr(converter, "_get_new_app_mode", MagicMock(return_value=AppMode.WORKFLOW)) - monkeypatch.setattr(converter, "_convert_to_app_config", MagicMock(return_value=app_config)) - monkeypatch.setattr( - converter, - "_convert_to_start_node", - MagicMock( - return_value={"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START, "variables": []}} - ), - ) - monkeypatch.setattr(converter, "_convert_to_knowledge_retrieval_node", MagicMock(return_value=None)) - monkeypatch.setattr( - converter, - "_convert_to_llm_node", - MagicMock(return_value={"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}}), - ) - monkeypatch.setattr( - converter, - "_convert_to_end_node", - MagicMock(return_value={"id": "end", "position": None, "data": {"type": BuiltinNodeTypes.END}}), - ) - workflow = converter.convert_app_model_config_to_workflow( - app_model=app_model, - app_model_config=_app_model_config(id="cfg"), - account_id="account-1", - session=sqlite_session, - ) - - graph = json.loads(workflow.graph) - node_ids = [node["id"] for node in graph["nodes"]] - assert node_ids == ["start", "llm", "end"] - - features = json.loads(workflow.features) - assert set(features.keys()) == {"text_to_speech", "file_upload", "sensitive_word_avoidance"} - assert sqlite_session.scalar(select(Workflow).where(Workflow.id == workflow.id)) is workflow - - -def test_convert_to_app_config_should_route_to_correct_manager( - converter: WorkflowConverter, - monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, -) -> None: - agent_result = SimpleNamespace(kind="agent") - chat_result = SimpleNamespace(kind="chat") - completion_result = SimpleNamespace(kind="completion") - agent_get_app_config = MagicMock(return_value=agent_result) - chat_get_app_config = MagicMock(return_value=chat_result) - completion_get_app_config = MagicMock(return_value=completion_result) - load_annotation_reply = MagicMock(return_value={"enabled": False}) - monkeypatch.setattr(converter_module.AgentChatAppConfigManager, "get_app_config", agent_get_app_config) - monkeypatch.setattr(converter_module.ChatAppConfigManager, "get_app_config", chat_get_app_config) - monkeypatch.setattr(converter_module.CompletionAppConfigManager, "get_app_config", completion_get_app_config) - monkeypatch.setattr(converter_module, "load_annotation_reply_config", load_annotation_reply) - agent_mode_app = _app_model(id="app-1", mode=AppMode.AGENT_CHAT, app_model_config_id="cfg-1") - agent_flag_app = _app_model(id="app-2", mode=AppMode.CHAT, app_model_config_id="cfg-2") - chat_app = _app_model(id="app-3", mode=AppMode.CHAT, app_model_config_id="cfg-3") - completion_app = _app_model(id="app-4", mode=AppMode.COMPLETION, app_model_config_id="cfg-4") - agent_mode_config = _app_model_config(id="cfg-1", app_id="app-1") - agent_flag_config = _app_model_config( - id="cfg-2", app_id="app-2", agent_mode=json.dumps({"enabled": True, "strategy": "react"}) - ) - chat_config = _app_model_config(id="cfg-3", app_id="app-3") - completion_config = _app_model_config(id="cfg-4", app_id="app-4") - sqlite_session.add_all( - [ - agent_mode_app, - agent_flag_app, - chat_app, - completion_app, - agent_mode_config, - agent_flag_config, - chat_config, - completion_config, - ] - ) - sqlite_session.commit() - - from_agent_mode = converter._convert_to_app_config( - app_model=agent_mode_app, - app_model_config=agent_mode_config, - session=sqlite_session, - ) - from_agent_flag = converter._convert_to_app_config( - app_model=agent_flag_app, - app_model_config=agent_flag_config, - session=sqlite_session, - ) - from_chat_mode = converter._convert_to_app_config( - app_model=chat_app, - app_model_config=chat_config, - session=sqlite_session, - ) - from_completion_mode = converter._convert_to_app_config( - app_model=completion_app, - app_model_config=completion_config, - session=sqlite_session, - ) - - assert from_agent_mode is agent_result - assert from_agent_flag is agent_result - assert from_chat_mode is chat_result - assert from_completion_mode is completion_result - load_annotation_reply.assert_has_calls( - [ - call(sqlite_session, "app-1"), - call(sqlite_session, "app-2"), - call(sqlite_session, "app-3"), - call(sqlite_session, "app-4"), - ] - ) - assert all( - manager_call.kwargs["annotation_reply"] == {"enabled": False} - for manager in (agent_get_app_config, chat_get_app_config, completion_get_app_config) - for manager_call in manager.call_args_list - ) - - -def test_convert_to_app_config_should_raise_for_invalid_app_mode( - converter: WorkflowConverter, unbound_session: Session -) -> None: - app_model = _app_model(mode=AppMode.WORKFLOW) - - with pytest.raises(ValueError, match="Invalid app mode"): - converter._convert_to_app_config( - app_model=app_model, app_model_config=_app_model_config(id="cfg"), session=unbound_session - ) - - def test_convert_to_http_request_node_should_skip_non_api_and_missing_extension_id( converter: WorkflowConverter, - unbound_session: Session, ) -> None: app_model = _app_model(id="app-1", tenant_id="tenant-1", mode=AppMode.CHAT) external_data_variables = [ @@ -707,10 +356,11 @@ def test_convert_to_http_request_node_should_skip_non_api_and_missing_extension_ ] nodes, mapping = converter._convert_to_http_request_node( - app_model=app_model, + app_id=app_model.id, + app_mode=AppMode(app_model.mode), variables=[], external_data_variables=external_data_variables, - session=unbound_session, + extensions={}, ) assert nodes == [] @@ -856,53 +506,13 @@ def test_replace_template_variables_should_replace_start_and_external_references assert result == "Hello {{#start.name#}} from {{#start.city#}} with {{#code_1.result#}}" -def test_graph_helpers_should_create_edges_append_nodes_and_choose_mode(converter: WorkflowConverter) -> None: +def test_graph_helpers_should_create_edges_and_append_nodes(converter: WorkflowConverter) -> None: graph = {"nodes": [{"id": "start", "position": None, "data": {"type": BuiltinNodeTypes.START}}], "edges": []} node = {"id": "llm", "position": None, "data": {"type": BuiltinNodeTypes.LLM}} edge = converter._create_edge("start", "llm") updated_graph = converter._append_node(graph, node) - workflow_mode = converter._get_new_app_mode(_app_model(mode=AppMode.COMPLETION)) - advanced_chat_mode = converter._get_new_app_mode(_app_model(mode=AppMode.CHAT)) assert edge == {"id": "start-llm", "source": "start", "target": "llm"} assert updated_graph["nodes"][-1]["id"] == "llm" assert updated_graph["edges"][-1]["source"] == "start" - assert workflow_mode == AppMode.WORKFLOW - assert advanced_chat_mode == AppMode.ADVANCED_CHAT - - -def test_get_api_based_extension_should_raise_when_extension_not_found( - converter: WorkflowConverter, - sqlite_session: Session, -) -> None: - with pytest.raises(ValueError, match="API Based Extension not found"): - converter._get_api_based_extension(tenant_id="tenant-1", api_based_extension_id="ext-1", session=sqlite_session) - - -def test_get_api_based_extension_should_return_entity_when_found( - converter: WorkflowConverter, - sqlite_session: Session, -) -> None: - extension = APIBasedExtension( - tenant_id="tenant-1", - name="API extension", - api_key="encrypted", - api_endpoint="https://example.com", - ) - extension.id = "ext-1" - decoy = APIBasedExtension( - tenant_id="other-tenant", - name="Other tenant API extension", - api_key="encrypted", - api_endpoint="https://example.com", - ) - decoy.id = "ext-other" - sqlite_session.add_all([extension, decoy]) - sqlite_session.commit() - - result = converter._get_api_based_extension( - tenant_id="tenant-1", api_based_extension_id="ext-1", session=sqlite_session - ) - - assert result is extension diff --git a/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py b/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py new file mode 100644 index 00000000000000..0c2eb491f01dcf --- /dev/null +++ b/api/tests/unit_tests/services/workflow/test_workflow_human_input_delivery.py @@ -0,0 +1,266 @@ +"""Exercise human input through the real console runtime and injected SQLite store.""" + +import json +from uuid import uuid4 + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.orm import Session, sessionmaker + +from extensions.application_services.human_input import build_human_input_debug_service +from factories.file_factory import builders +from graphon.variables import StringVariable +from models.human_input import HumanInputForm, HumanInputFormRecipient +from models.human_input_delivery import EmailDeliveryConfig, EmailDeliveryMethod, EmailRecipients, ExternalRecipient +from models.human_input_entities import ( + FileInputConfig, + HumanInputNodeData, + ParagraphInputConfig, + UserActionConfig, +) +from models.workflow import Workflow, WorkflowDraftVariable +from repositories.human_input import form_repository +from services import human_input_delivery_test_service as delivery_module +from services import workflow_service as workflow_module +from services.entities.feature_entities import FeatureModel +from services.human_input.debug_service import HumanInputDebugService +from services.workflow.runtime_gateway import WorkflowRuntimeGateway +from tests.helpers.unit.model_factories import make_upload_file +from tests.unit_tests.services.workflow.test_runtime_gateway import CONTEXT, RecordingSession, runtime_gateway # noqa: F401 + + +@pytest.fixture +def human_debug( + runtime: tuple[WorkflowRuntimeGateway, list[RecordingSession]], +) -> tuple[HumanInputDebugService, list[RecordingSession]]: + gateway, sessions = runtime + return build_human_input_debug_service(database_client=gateway._sessions), sessions + + +def save_human_input_draft(factory: sessionmaker[Session], *, enabled: bool, debug: bool) -> str: + method = EmailDeliveryMethod( + id=uuid4(), + enabled=enabled, + config=EmailDeliveryConfig( + recipients=EmailRecipients( + include_bound_group=False, items=[ExternalRecipient(email="recipient@example.com")] + ), + subject="Test {{recipient_email}}", + body="{{form_content}} {{#url#}}", + debug_mode=debug, + ), + ) + node = HumanInputNodeData( + title="Human Input", + delivery_methods=[method], + form_content="{{#conversation.greeting#}} {{#upstream.output#}}", + inputs=[ParagraphInputConfig(output_variable_name="answer")], + user_actions=[UserActionConfig(id="approve", title="Approve")], + ) + with factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "human", + "data": { + **node.model_dump(mode="json"), + "type": "human-input", + }, + } + ], + "edges": [], + } + ) + draft.conversation_variables = [StringVariable(name="greeting", value="Hello")] + return str(method.id) + + +def forbid_global_database(monkeypatch: pytest.MonkeyPatch) -> None: + # Fail if any step falls back to a process-global database instead of the + # injected factory, including form persistence and email member lookup. + monkeypatch.setattr(workflow_module, "db", object()) + monkeypatch.setattr(delivery_module, "db", object()) + monkeypatch.setattr(form_repository, "session_factory", object()) + monkeypatch.setattr(builders, "session_factory", object()) + + +def test_preview_and_submission_use_injected_database( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = human_debug + save_human_input_draft(sqlite_session_factory, enabled=True, debug=False) + forbid_global_database(monkeypatch) + preview = gateway.preview_form(CONTEXT, "app-1", "human", {"#upstream.output#": "world"}) + assert preview["form_content"] == "Hello world" + assert sessions + assert all(session.closed for session in sessions) + outputs = gateway.submit_form( + CONTEXT, + "app-1", + "human", + inputs={"#upstream.output#": "world"}, + form_inputs={"answer": "yes"}, + action="approve", + ) + assert outputs["answer"] == "yes" + assert outputs["__action_value"] == "Approve" + assert all(session.closed for session in sessions) + with sqlite_session_factory() as session: + variables = session.scalars( + select(WorkflowDraftVariable).where( + WorkflowDraftVariable.app_id == "app-1", WorkflowDraftVariable.node_id == "human" + ) + ).all() + assert {variable.name: variable.get_value().value for variable in variables}["answer"] == "yes" + + +@pytest.mark.parametrize("submit", [False, True]) +def test_debug_secret_is_decrypted_after_repository_session_closes( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + submit: bool, +) -> None: + service, sessions = human_debug + save_human_input_draft(sqlite_session_factory, enabled=True, debug=False) + with sqlite_session_factory.begin() as session: + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + graph = dict(draft.graph_dict) + graph["nodes"][0]["data"]["form_content"] = "Token: {{#env.KEY#}}" + draft.graph = json.dumps(graph) + draft._environment_variables = json.dumps( + {"KEY": {"id": "secret-id", "name": "KEY", "value_type": "secret", "value": "wrapped"}} + ) + unwrapped: list[str] = [] + + def decrypt(*, tenant_id: str, token: str) -> str: + assert tenant_id == CONTEXT.active_workspace_id + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + unwrapped.append(token) + return "plaintext" + + monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt) + if submit: + result = service.submit_form( + CONTEXT, "app-1", "human", inputs={}, form_inputs={"answer": "yes"}, action="approve" + ) + assert result["__rendered_content"] == "Token: plaintext" + else: + result = service.preview_form(CONTEXT, "app-1", "human", {}) + assert result["form_content"] == "Token: plaintext" + assert unwrapped == ["wrapped"] + + +@pytest.mark.parametrize(("enabled", "debug"), [(True, False), (False, False), (True, True)]) +def test_email_delivery_commits_form_and_closes_sessions_before_external_send( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, + enabled: bool, + debug: bool, +) -> None: + gateway, sessions = human_debug + method_id = save_human_input_draft(sqlite_session_factory, enabled=enabled, debug=debug) + forbid_global_database(monkeypatch) + monkeypatch.setattr( + delivery_module.FeatureService, + "get_features", + lambda *_args, **_kwargs: FeatureModel(human_input_email_delivery_enabled=True), + ) + monkeypatch.setattr(delivery_module.mail, "is_inited", lambda: True) + sent: list[str] = [] + recipient = "test@example.com" if debug else "recipient@example.com" + + def send(*, to: str, subject: str, html: str) -> None: + assert sessions + assert all(session.closed and not session.in_transaction() for session in sessions) + # A separate reader sees the committed form and token before mail leaves. + with sqlite_session_factory() as session: + form = session.scalar(select(HumanInputForm).where(HumanInputForm.app_id == "app-1")) + assert form is not None + assert form.rendered_content == "Hello world" + saved_recipient = session.scalar( + select(HumanInputFormRecipient).where(HumanInputFormRecipient.form_id == form.id) + ) + assert saved_recipient is not None + assert saved_recipient.access_token + assert saved_recipient.access_token in html + assert to == recipient + assert subject == f"Test {recipient}" + assert "Hello world" in html + sent.append(to) + + monkeypatch.setattr(delivery_module.mail, "send", send) + gateway.test_delivery( + CONTEXT, "app-1", "human", inputs={"#upstream.output#": "world"}, delivery_method_id=method_id + ) + assert sent == [recipient] + + +def test_human_input_requires_draft( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], +) -> None: + gateway, sessions = human_debug + with sqlite_session_factory.begin() as session: + session.execute(delete(Workflow)) + with pytest.raises(ValueError, match="Workflow not initialized"): + gateway.test_delivery(CONTEXT, "app-1", "human", inputs={}, delivery_method_id="missing") + assert all(session.closed for session in sessions) + + +def test_file_preview_and_submission_load_upload_from_injected_database( + human_debug: tuple[HumanInputDebugService, list[RecordingSession]], + sqlite_session_factory: sessionmaker[Session], + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway, sessions = human_debug + file_id = str(uuid4()) + node = HumanInputNodeData( + title="Attachment", + form_content="{{#upstream.file#}}", + inputs=[FileInputConfig(output_variable_name="attachment")], + user_actions=[UserActionConfig(id="approve", title="Approve")], + ) + with sqlite_session_factory.begin() as session: + session.add(make_upload_file(file_id=file_id)) + draft = session.scalar(select(Workflow).where(Workflow.version == Workflow.VERSION_DRAFT)) + assert draft is not None + draft.graph = json.dumps( + { + "nodes": [ + { + "id": "human", + "data": { + **node.model_dump(mode="json"), + "type": "human-input", + }, + } + ], + "edges": [], + } + ) + forbid_global_database(monkeypatch) + file_mapping = {"type": "document", "transfer_method": "local_file", "upload_file_id": file_id} + preview = gateway.preview_form(CONTEXT, "app-1", "human", {"#upstream.file#": file_mapping}) + assert "test.txt" in str(preview["form_content"]) + outputs = gateway.submit_form( + CONTEXT, + "app-1", + "human", + inputs={"#upstream.file#": file_mapping}, + form_inputs={"attachment": file_mapping}, + action="approve", + ) + attachment = outputs["attachment"] + assert isinstance(attachment, dict) + assert attachment["filename"] == "test.txt" + assert sessions + assert all(session.closed for session in sessions) diff --git a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py index 6eb7b78b02710f..b75dd60eefa7dc 100644 --- a/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py +++ b/api/tests/unit_tests/services/workspace/test_resource_cleanup_service.py @@ -276,7 +276,7 @@ def test_direct_cleanup_does_not_resolve_a_console_login( _service(sqlite_session_factory).cleanup("target") sqlite_session.expire_all() assert sqlite_session.get(App, "app") is None - assert retirement.retire_unowned.call_args.kwargs["account_id"] is None + retirement.return_value.retire_unowned.assert_called_once_with(tenant_id="target", agent_ids=set(), account_id=None) user.__bool__.assert_not_called() diff --git a/api/tests/unit_tests/tasks/app_generate/test_workflow_execute_task.py b/api/tests/unit_tests/tasks/app_generate/test_workflow_execute_task.py index 8532cb84f5c6ba..07d3e9c53ddc70 100644 --- a/api/tests/unit_tests/tasks/app_generate/test_workflow_execute_task.py +++ b/api/tests/unit_tests/tasks/app_generate/test_workflow_execute_task.py @@ -16,7 +16,6 @@ from sqlalchemy.orm import Session, sessionmaker from core.app.app_config.entities import WorkflowUIBasedAppConfig -from core.app.apps.workflow.app_generator import WorkflowAppGenerator from core.app.entities.app_invoke_entities import ( AdvancedChatAppGenerateEntity, InvokeFrom, @@ -41,6 +40,8 @@ WorkflowRunNotPausedError, _WorkflowRunError, ) +from services.workflow.execution.adapters.workflow.app_generator import WorkflowAppGenerator +from services.workflow.variable_contracts import WorkflowExecutionVariables from tasks import async_workflow_tasks from tasks.app_generate import workflow_execute_task as workflow_execute_task_module from tasks.app_generate.workflow_execute_task import ( @@ -89,6 +90,33 @@ def _single_event_generator(payload): yield payload +@pytest.mark.parametrize("app_mode", [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]) +def test_worker_passes_its_binding_resolver_to_generation( + monkeypatch: pytest.MonkeyPatch, + sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, + app_mode: AppMode, +) -> None: + app = _make_app() + app.mode = app_mode + workflow, user = _make_workflow(), _make_account() + params = AppExecutionParams.new(app, workflow, user, {}, InvokeFrom.DEBUGGER) + runner = _AppRunner(sqlite_session_factory, params, variables=workflow_variables) + generator = MagicMock() + monkeypatch.setattr( + workflow_execute_task_module, + "WorkflowAppGenerator" if app_mode == AppMode.WORKFLOW else "AdvancedChatAppGenerator", + generator, + ) + runner._run_app(app=app, workflow=workflow, user=user, pause_state_config=MagicMock()) + assert generator.call_args.kwargs["runtime"] is runner._runtime + assert runner._runtime.agent_bindings._sessions is sqlite_session_factory + if app_mode == AppMode.WORKFLOW: + execution_identity = generator.return_value.generate.call_args.kwargs["execution_identity"] + assert execution_identity.user_id == user.id + assert execution_identity.user_from == UserFrom.ACCOUNT + + def _decode_published_payload(payload: bytes) -> dict[str, object] | str: return json.loads(payload.decode()) @@ -368,13 +396,13 @@ def test_get_error_message(event: str | Mapping[str, object] | BaseModel, expect def mock_topic(monkeypatch: pytest.MonkeyPatch) -> MagicMock: topic = MagicMock() monkeypatch.setattr( - "tasks.app_generate.workflow_execute_task.MessageBasedAppGenerator.get_response_topic", + "tasks.app_generate.workflow_execute_task.WorkflowEventStream.get_response_topic", lambda *_args, **_kwargs: topic, ) return topic -def test_publish_streaming_response_with_uuid(mock_topic: MagicMock): +def test_publish_streaming_response_with_uuid(mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session]): workflow_run_id = uuid.uuid4() response_stream = iter( [ @@ -390,13 +418,19 @@ def test_publish_streaming_response_with_uuid(mock_topic: MagicMock): workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) assert [payload["event"] for payload in payloads] == ["workflow_started", "workflow_finished"] -def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): +def test_publish_streaming_response_coerces_string_uuid( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): workflow_run_id = uuid.uuid4() response_stream = iter([{"event": "workflow_paused", "task_id": "task-id"}]) @@ -407,6 +441,10 @@ def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -414,7 +452,7 @@ def test_publish_streaming_response_coerces_string_uuid(mock_topic: MagicMock): def test_publish_streaming_response_publishes_started_then_failed_terminal_when_iteration_raises( - mock_topic: MagicMock, + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] ): def _response_stream(): if False: @@ -429,6 +467,10 @@ def _response_stream(): workflow_id="workflow-id", inputs={"foo": "bar"}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -440,8 +482,7 @@ def _response_stream(): def test_publish_streaming_response_recovers_when_workflow_started_publish_fails_first( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter([{"event": "workflow_started", "task_id": "task-id"}]) @@ -468,6 +509,10 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={"file": object()}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] @@ -481,8 +526,7 @@ def _publish(payload: bytes) -> None: def test_publish_streaming_response_publishes_failed_terminal_without_duplicate_started_on_publish_error( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -514,6 +558,10 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] @@ -525,8 +573,7 @@ def _publish(payload: bytes) -> None: def test_publish_streaming_response_recovers_when_workflow_finished_publish_fails_first( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.ERROR, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -558,19 +605,21 @@ def _publish(payload: bytes) -> None: workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) assert [payload["event"] for payload in successful_payloads] == ["workflow_started", "workflow_finished"] assert successful_payloads[1]["task_id"] == "task-id" - assert successful_payloads[1]["data"]["status"] == WorkflowExecutionStatus.FAILED - assert successful_payloads[1]["data"]["error"] == "finished publish failed" + assert successful_payloads[1]["data"]["status"] == WorkflowExecutionStatus.SUCCEEDED assert "workflow-run-id" in caplog.text - assert "publishing fallback terminal event" in caplog.text + assert "Retrying engine terminal event publication" in caplog.text def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_without_terminal_event( - mock_topic: MagicMock, - caplog: pytest.LogCaptureFixture, + mock_topic: MagicMock, caplog: pytest.LogCaptureFixture, sqlite_session_factory: sessionmaker[Session] ): caplog.set_level(logging.WARNING, logger="tasks.app_generate.workflow_execute_task") response_stream = iter( @@ -591,6 +640,10 @@ def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_with workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -602,7 +655,9 @@ def test_publish_streaming_response_publishes_failed_terminal_on_exhaustion_with assert "ended without a terminal event" in caplog.text -def test_publish_streaming_response_uses_error_message_for_failed_terminal(mock_topic: MagicMock): +def test_publish_streaming_response_uses_error_message_for_failed_terminal( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): def response_stream() -> Generator[str | Mapping[str, object] | BaseModel]: yield { "event": "error", @@ -619,6 +674,10 @@ def response_stream() -> Generator[str | Mapping[str, object] | BaseModel]: workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -634,7 +693,9 @@ def response_stream() -> Generator[str | Mapping[str, object] | BaseModel]: assert finished_data["error"] == "LLM provider and model are required." -def test_publish_streaming_response_does_not_publish_synthetic_failure_after_terminal_event(mock_topic: MagicMock): +def test_publish_streaming_response_does_not_publish_synthetic_failure_after_terminal_event( + mock_topic: MagicMock, sqlite_session_factory: sessionmaker[Session] +): response_stream = iter( [ { @@ -673,6 +734,10 @@ def test_publish_streaming_response_does_not_publish_synthetic_failure_after_ter workflow_id="workflow-id", inputs={}, started_reason=WorkflowStartReason.INITIAL, + agent_snapshot=False, + sessions=sqlite_session_factory, + tenant_id="tenant-id", + app_id="app-id", ) payloads = _published_payloads(mock_topic) @@ -683,6 +748,8 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin mock_topic: MagicMock, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -696,7 +763,7 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) monkeypatch.setattr(runner, "_resolve_user", _make_account) monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) @@ -732,7 +799,7 @@ def test_app_runner_streaming_failure_publishes_started_then_failed_workflow_fin def test_app_runner_resolves_account_without_switching_tenant( - sqlite_session_factory: sessionmaker[Session], + sqlite_session_factory: sessionmaker[Session], *, workflow_variables: WorkflowExecutionVariables ): account_id = str(uuid.uuid4()) exec_params = AppExecutionParams( @@ -751,7 +818,7 @@ def test_app_runner_resolves_account_without_switching_tenant( account.id = account_id session.add(account) session.commit() - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) resolved_user = runner._resolve_user() @@ -777,6 +844,8 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior mock_topic: MagicMock, monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -790,13 +859,13 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) monkeypatch.setattr(runner, "_resolve_user", _make_account) monkeypatch.setattr(runner, "_setup_flask_context", lambda _user: nullcontext()) monkeypatch.setattr(runner, "_run_app", lambda **_kwargs: (_ for _ in ()).throw(ValueError("Invalid upload file"))) monkeypatch.setattr( - "core.workflow.workflow_entry.WorkflowEntry.handle_special_values", + "services.workflow.execution.adapters.workflow_entry.WorkflowEntry.handle_special_values", lambda value: (_ for _ in ()).throw(AssertionError("pre-runtime helper should not normalize inputs")), ) @@ -811,6 +880,8 @@ def test_app_runner_streaming_failure_keeps_existing_pre_runtime_helper_behavior def test_app_runner_streaming_success_calls_publish_streaming_response_with_full_signature( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): exec_params = AppExecutionParams( app_id="app-id", @@ -824,7 +895,7 @@ def test_app_runner_streaming_success_calls_publish_streaming_response_with_full workflow_run_id="workflow-run-id", ) _persist_app_and_workflow(sqlite_session_factory) - runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params) + runner = _AppRunner(session_factory=sqlite_session_factory, exec_params=exec_params, variables=workflow_variables) response_stream = _single_event_generator({"event": "message"}) publish_streaming_response = MagicMock() @@ -845,6 +916,12 @@ def test_app_runner_streaming_success_calls_publish_streaming_response_with_full exec_params.workflow_id, exec_params.args.get("inputs", {}), WorkflowStartReason.INITIAL, + sessions=sqlite_session_factory, + tenant_id=exec_params.tenant_id, + app_id="app-id", + agent_snapshot=False, + on_finalized=runner._mark_execution_finalized, + on_terminal=None, ) @@ -852,6 +929,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run( monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): workflow_run_id = "run-id" conversation_id = "conversation-id" @@ -893,7 +971,7 @@ def test_resume_app_execution_queries_message_by_conversation_and_workflow_run( monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", resume_advanced_chat) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_workflow", MagicMock()) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) workflow_run_repo.resume_workflow_pause.assert_called_once_with(workflow_run_id, pause_entity, before_resume=ANY) resume_advanced_chat.assert_called_once() @@ -905,6 +983,7 @@ def test_resume_app_execution_returns_early_when_advanced_chat_missing_conversat monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): workflow_run_id = "run-id" _persist_resumption_models(sqlite_session_factory, workflow_run_id=workflow_run_id) @@ -940,7 +1019,7 @@ def test_resume_app_execution_returns_early_when_advanced_chat_missing_conversat resume_advanced_chat = MagicMock() monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", resume_advanced_chat) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) workflow_run_repo.resume_workflow_pause.assert_not_called() resume_advanced_chat.assert_not_called() @@ -950,6 +1029,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming( monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): """A resumed run reuses the paused task ID, so it must not inherit its cancellation signals. @@ -995,7 +1075,7 @@ def test_resume_app_execution_clears_stale_cancellation_signals_before_resuming( monkeypatch.setattr("tasks.app_generate.workflow_execute_task.clear_app_task_cancellation_signals", clear_signals) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_workflow", resume_workflow) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) clear_signals.assert_called_once_with(generate_entity.task_id) # Clearing after the engine started would let it observe the stale abort first. @@ -1006,6 +1086,7 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandone monkeypatch: pytest.MonkeyPatch, sqlite_engine: Engine, sqlite_session_factory: sessionmaker[Session], + workflow_variables: WorkflowExecutionVariables, ): """No attempt is starting, so nothing may clear the signals guarding this task.""" workflow_run_id = "run-id" @@ -1044,7 +1125,7 @@ def test_resume_app_execution_keeps_cancellation_signals_when_resume_is_abandone monkeypatch.setattr("tasks.app_generate.workflow_execute_task.clear_app_task_cancellation_signals", clear_signals) monkeypatch.setattr("tasks.app_generate.workflow_execute_task._resume_advanced_chat", MagicMock()) - _resume_app_execution({"workflow_run_id": workflow_run_id}) + _resume_app_execution({"workflow_run_id": workflow_run_id}, variables=workflow_variables) clear_signals.assert_not_called() @@ -1071,6 +1152,8 @@ def test_resume_recovers_legacy_task_before_claiming_pause( sqlite_session_factory: sessionmaker[Session], entry_point: str, scenario: str, + *, + workflow_variables: WorkflowExecutionVariables, ) -> None: workflow_run_id = str(uuid.uuid4()) task_id = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" @@ -1180,8 +1263,10 @@ def record_resume_pause( if entry_point == "async" and scenario == "concurrent-stop": with pytest.raises(WorkflowRunNotPausedError): invoke({"workflow_run_id": workflow_run_id}) - else: + elif entry_point == "async": invoke({"workflow_run_id": workflow_run_id}) + else: + invoke({"workflow_run_id": workflow_run_id}, variables=workflow_variables) with sqlite_session_factory() as session: persisted = session.get(WorkflowRun, workflow_run_id) @@ -1209,8 +1294,9 @@ def record_resume_pause( def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( monkeypatch: pytest.MonkeyPatch, - sqlite_session: Session, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_advanced_chat_generate_entity(conversation_id="conversation-id") generate_entity.stream = False @@ -1221,7 +1307,7 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.AdvancedChatAppGenerator", - lambda *, execution_driver: generator_instance, + lambda *, execution_driver, **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1249,12 +1335,12 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( pause_state_config=MagicMock(), workflow_run_id="workflow-run-id", workflow_run=_make_workflow_run(), - session=sqlite_session, + variables=workflow_variables, ) resumed_entity = generator_instance.resume.call_args.kwargs["application_generate_entity"] assert resumed_entity.stream is True - assert generator_instance.resume.call_args.kwargs["session"] is sqlite_session + assert "session" not in generator_instance.resume.call_args.kwargs publish_streaming_response.assert_called_once_with( response_stream, "workflow-run-id", @@ -1262,12 +1348,18 @@ def test_resume_advanced_chat_publishes_events_for_originally_blocking_runs( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, ) def test_resume_workflow_publishes_events_for_originally_blocking_runs( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_workflow_generate_entity(stream=False) workflow = _make_workflow() @@ -1277,7 +1369,7 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.WorkflowAppGenerator", - lambda *, execution_driver: generator_instance, + lambda *, execution_driver, **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1308,6 +1400,7 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( workflow_run=_make_workflow_run(), workflow_run_repo=workflow_run_repo, pause_entity=pause_entity, + variables=workflow_variables, ) resumed_entity = generator_instance.resume.call_args.kwargs["application_generate_entity"] @@ -1319,6 +1412,11 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, + on_terminal=None, ) workflow_run_repo.delete_workflow_pause.assert_called_once_with(pause_entity) @@ -1326,6 +1424,8 @@ def test_resume_workflow_publishes_events_for_originally_blocking_runs( def test_resume_workflow_ignores_missing_old_pause_after_repause( monkeypatch: pytest.MonkeyPatch, sqlite_session_factory: sessionmaker[Session], + *, + workflow_variables: WorkflowExecutionVariables, ): generate_entity = _build_workflow_generate_entity(stream=False) workflow = _make_workflow() @@ -1335,7 +1435,7 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( generator_instance.resume.return_value = response_stream monkeypatch.setattr( "tasks.app_generate.workflow_execute_task.WorkflowAppGenerator", - lambda *, execution_driver: generator_instance, + lambda *, execution_driver, **_dependencies: generator_instance, ) publish_streaming_response = MagicMock() @@ -1367,6 +1467,7 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( workflow_run=_make_workflow_run(), workflow_run_repo=workflow_run_repo, pause_entity=pause_entity, + variables=workflow_variables, ) publish_streaming_response.assert_called_once_with( @@ -1376,5 +1477,10 @@ def test_resume_workflow_ignores_missing_old_pause_after_repause( workflow.id, generate_entity.inputs, WorkflowStartReason.RESUMPTION, + sessions=sqlite_session_factory, + tenant_id=workflow.tenant_id, + app_id=workflow.app_id, + agent_snapshot=False, + on_terminal=None, ) workflow_run_repo.delete_workflow_pause.assert_called_once_with(pause_entity) diff --git a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py index 6ab9f43d8611fc..ccd582d9a89c0d 100644 --- a/api/tests/unit_tests/tasks/test_async_workflow_tasks.py +++ b/api/tests/unit_tests/tasks/test_async_workflow_tasks.py @@ -1,8 +1,174 @@ import ast import inspect +from types import SimpleNamespace +from unittest.mock import MagicMock +import pytest + +from core.app.app_config.entities import WorkflowUIBasedAppConfig +from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom, WorkflowAppGenerateEntity +from models.enums import AppTriggerType, WorkflowRunTriggeredFrom +from models.model import AppMode +from repositories.workflow.definition_repository import workflow_snapshot from services.errors.app import WorkflowNotFoundError +from services.workflow.contracts import WorkflowSnapshot +from services.workflow.entities import TriggerData, WorkflowTaskData from tasks import async_workflow_tasks +from tests.helpers.unit.model_factories import make_account, make_app, make_workflow + + +def _session_context(session: MagicMock) -> MagicMock: + context = MagicMock() + context.__enter__.return_value = session + context.__exit__.return_value = False + return context + + +def test_execute_workflow_projects_the_persisted_workflow_before_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + workflow_model = make_workflow(workflow_id="workflow-1", tenant_id="tenant-1", app_id="app-1") + app = make_app(app_id="app-1", tenant_id="tenant-1", workflow_id=workflow_model.id) + user = make_account(account_id="account-1") + trigger_data = TriggerData( + app_id=app.id, + tenant_id=app.tenant_id, + workflow_id=workflow_model.id, + root_node_id="start", + inputs={"question": "hello"}, + trigger_type=AppTriggerType.TRIGGER_SCHEDULE, + trigger_from=WorkflowRunTriggeredFrom.SCHEDULE, + ) + trigger_log = SimpleNamespace( + id="trigger-log-1", + app_id=app.id, + workflow_id=workflow_model.id, + trigger_data=trigger_data.model_dump_json(), + status=None, + ) + + session = MagicMock() + session.scalar.side_effect = [app, workflow_model] + task_session_factory = MagicMock() + task_session_factory.create_session.return_value = _session_context(session) + task_session_factory.get_session_maker.return_value = MagicMock() + trigger_log_repository = MagicMock() + trigger_log_repository.get_by_id.return_value = trigger_log + generator = MagicMock() + + monkeypatch.setattr(async_workflow_tasks, "session_factory", task_session_factory) + monkeypatch.setattr( + async_workflow_tasks, + "SQLAlchemyWorkflowTriggerLogRepository", + MagicMock(return_value=trigger_log_repository), + ) + monkeypatch.setattr(async_workflow_tasks, "_get_user", MagicMock(return_value=user)) + monkeypatch.setattr(async_workflow_tasks, "build_workflow_execution_dependencies", MagicMock()) + monkeypatch.setattr(async_workflow_tasks, "WorkflowAppGenerator", MagicMock(return_value=generator)) + monkeypatch.setattr(async_workflow_tasks, "TriggerPostLayer", MagicMock(return_value=MagicMock())) + + async_workflow_tasks._execute_workflow_common( + WorkflowTaskData(workflow_trigger_log_id=trigger_log.id), + MagicMock(), + MagicMock(), + ) + + execution_workflow = generator.generate.call_args.kwargs["workflow"] + assert isinstance(execution_workflow, WorkflowSnapshot) + assert execution_workflow == workflow_snapshot(workflow_model) + assert execution_workflow is not workflow_model + + +def test_resume_workflow_projects_the_persisted_workflow_before_resuming( + monkeypatch: pytest.MonkeyPatch, +) -> None: + workflow_model = make_workflow(workflow_id="workflow-1", tenant_id="tenant-1", app_id="app-1") + app = make_app(app_id="app-1", tenant_id="tenant-1", workflow_id=workflow_model.id) + user = make_account(account_id="account-1") + workflow_run = SimpleNamespace( + id="workflow-run-1", + task_id="task-1", + workflow_id=workflow_model.id, + app_id=app.id, + triggered_from=WorkflowRunTriggeredFrom.SCHEDULE, + ) + pause_entity = MagicMock(id="pause-1", workflow_execution_id=workflow_run.id) + pause_entity.get_state.return_value = b"serialized-resumption-context" + workflow_run_repository = MagicMock() + workflow_run_repository.get_workflow_pause.return_value = pause_entity + workflow_run_repository.get_workflow_run_by_id_without_tenant.return_value = workflow_run + + generate_entity = WorkflowAppGenerateEntity( + task_id="task-1", + app_config=WorkflowUIBasedAppConfig( + tenant_id=app.tenant_id, + app_id=app.id, + app_mode=AppMode.WORKFLOW, + workflow_id=workflow_model.id, + ), + inputs={}, + files=[], + user_id=user.id, + user_from=UserFrom.ACCOUNT, + stream=False, + invoke_from=InvokeFrom.WEB_APP, + workflow_execution_id=workflow_run.id, + ) + resumption_context = MagicMock() + resumption_context.get_generate_entity.return_value = generate_entity + resumption_context.serialized_graph_runtime_state = "serialized-runtime-state" + response_stream_filter = MagicMock() + resumption_context.get_response_stream_filter.return_value = response_stream_filter + graph_runtime_state = MagicMock() + + session = MagicMock() + session.scalar.side_effect = [workflow_model, app] + resumed_session_factory = MagicMock() + resumed_session_factory.return_value = _session_context(session) + generator = MagicMock() + execution_repository = MagicMock() + node_execution_repository = MagicMock() + + monkeypatch.setattr(async_workflow_tasks, "db", SimpleNamespace(engine=object())) + monkeypatch.setattr(async_workflow_tasks, "sessionmaker", MagicMock(return_value=resumed_session_factory)) + monkeypatch.setattr( + async_workflow_tasks.DifyAPIRepositoryFactory, + "create_api_workflow_run_repository", + MagicMock(return_value=workflow_run_repository), + ) + monkeypatch.setattr( + async_workflow_tasks.WorkflowResumptionContext, + "loads", + MagicMock(return_value=resumption_context), + ) + monkeypatch.setattr( + async_workflow_tasks.RuntimeState, + "from_snapshot", + MagicMock(return_value=graph_runtime_state), + ) + monkeypatch.setattr(async_workflow_tasks, "_get_user", MagicMock(return_value=user)) + monkeypatch.setattr( + async_workflow_tasks.DifyCoreRepositoryFactory, + "create_workflow_execution_repository", + MagicMock(return_value=execution_repository), + ) + monkeypatch.setattr( + async_workflow_tasks.DifyCoreRepositoryFactory, + "create_workflow_node_execution_repository", + MagicMock(return_value=node_execution_repository), + ) + monkeypatch.setattr(async_workflow_tasks, "build_workflow_execution_dependencies", MagicMock()) + monkeypatch.setattr(async_workflow_tasks, "WorkflowAppGenerator", MagicMock(return_value=generator)) + monkeypatch.setattr(async_workflow_tasks, "_query_trigger_log_info", MagicMock(return_value=None)) + + async_workflow_tasks.resume_workflow_execution.run({"workflow_run_id": workflow_run.id}) + + resumed_workflow = generator.resume.call_args.kwargs["workflow"] + assert isinstance(resumed_workflow, WorkflowSnapshot) + assert resumed_workflow == workflow_snapshot(workflow_model) + assert resumed_workflow is not workflow_model + workflow_run_repository.resume_workflow_pause.assert_called_once_with(workflow_run.id, pause_entity) + workflow_run_repository.delete_workflow_pause.assert_called_once_with(pause_entity) def test_workflow_not_found_error_message_is_a_single_formatted_string() -> None: diff --git a/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py b/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py index 5db2ddba5dedbb..731148159ac8d6 100644 --- a/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py +++ b/api/tests/unit_tests/tasks/test_human_input_timeout_tasks.py @@ -9,10 +9,10 @@ from sqlalchemy.orm import Session, sessionmaker import core.db.session_factory as session_factory_module -from core.repositories.human_input_repository import HumanInputFormSubmissionRepository from enums.human_input import HumanInputFormKind, HumanInputFormStatus from models.human_input import HumanInputForm from models.human_input_entities import FormDefinition +from repositories.human_input.form_repository import HumanInputFormSubmissionRepository from tasks import human_input_timeout_tasks as task_module from tests.helpers.unit.config_override import apply_config_overrides diff --git a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py index 889958198eece2..29032721b86bc2 100644 --- a/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py +++ b/api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py @@ -11,12 +11,13 @@ import tasks.remove_app_and_related_data_task as remove_app_task_module from enums import DeploymentEdition +from enums.agent import WorkflowAgentBindingType from extensions.ext_application_services import ApplicationServices from graphon.enums import WorkflowExecutionStatus from graphon.variables.segments import StringSegment from libs.archive_storage import ArchiveStorageNotConfiguredError from models import AppStar -from models.agent import WorkflowAgentBindingType, WorkflowAgentNodeBinding +from models.agent import WorkflowAgentNodeBinding from models.enums import CreatorUserRole, WorkflowRunTriggeredFrom from models.workflow import WorkflowArchiveLog, WorkflowDraftVariable from services.network_access_group_service import NetworkAccessGroupUpstreamError diff --git a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py index e68ef04dc31930..1e7e0a2c3784be 100644 --- a/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py +++ b/api/tests/unit_tests/tasks/test_retry_document_indexing_task.py @@ -4,16 +4,18 @@ import pytest from sqlalchemy.orm import Session -from core.app.apps.pipeline.pipeline_generator import PipelineGenerator from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType from enums import DeploymentEdition from models import Account, Tenant, TenantAccountJoin from models.account import TenantAccountRole from models.dataset import Dataset, Document from models.enums import DatasetRuntimeMode, DataSourceType, DocumentCreatedFrom, IndexingStatus +from services.workflow.execution.adapters.pipeline.pipeline_generator import PipelineGenerator from tasks.retry_document_indexing_task import retry_document_indexing_task from tests.helpers.unit.config_override import config_overrides_context +pytestmark = pytest.mark.usefixtures("workflow_application") + @config_overrides_context(DEPLOYMENT_EDITION=DeploymentEdition.COMMUNITY) @pytest.mark.parametrize("runtime_mode", [DatasetRuntimeMode.GENERAL, DatasetRuntimeMode.RAG_PIPELINE]) diff --git a/packages/contracts/generated/api/console/apps/errors.gen.ts b/packages/contracts/generated/api/console/apps/errors.gen.ts index 4aa9548e94fe18..3652108ea43858 100644 --- a/packages/contracts/generated/api/console/apps/errors.gen.ts +++ b/packages/contracts/generated/api/console/apps/errors.gen.ts @@ -604,20 +604,9 @@ export const get4Errors = { app_not_found: { status: 404, data: zAppNotFoundErro export const get5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post4Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post5Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post6Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post7Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post8Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post9Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post9Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const post10Errors = { agent_not_bound: { status: 400, data: zAgentNotBoundError }, @@ -791,8 +780,6 @@ export const post22Errors = { export const get30Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post23Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get31Errors = { roster_agent_package_export_failed: { status: 500, data: zRosterAgentPackageExportFailedError }, } @@ -971,8 +958,6 @@ export const get50Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post41Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get52Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const get53Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } @@ -1061,28 +1046,12 @@ export const get63Errors = { app_not_found: { status: 404, data: zAppNotFoundErr export const get64Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const get65Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get66Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get68Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } -export const post48Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post49Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post50Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post51Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post52Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } -export const post53Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post54Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get69Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } export const post55Errors = { @@ -1128,16 +1097,7 @@ export const put5Errors = { plaintext_secret_not_allowed_error: { status: 400, data: zPlaintextSecretNotAllowedError }, } -export const get71Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post59Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post60Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post61Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post61Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get73Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, @@ -1162,34 +1122,22 @@ export const get76Errors = { workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, } -export const post62Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post62Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } -export const post63Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, -} +export const post63Errors = { rate_limit_error: { status: 429, data: zRateLimitError } } export const get79Errors = { draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const get80Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, } export const post64Errors = { - app_not_found: { status: 404, data: zAppNotFoundError }, draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, } -export const get81Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const post65Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const get82Errors = { app_not_found: { status: 404, data: zAppNotFoundError }, workflow_run_not_found: { status: 404, data: zWorkflowRunNotFoundError }, @@ -1218,14 +1166,6 @@ export const get86Errors = { webhook_trigger_not_found: { status: 404, data: zWebhookTriggerNotFoundError }, } -export const post66Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const delete14Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const patch3Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - -export const get87Errors = { app_not_found: { status: 404, data: zAppNotFoundError } } - export const post67Errors = { agent_not_published: { status: 409, data: zAgentNotPublishedError }, max_keys_exceeded: { status: 400, data: zMaxKeysExceededError }, diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index f22bf4f76a1e38..64c9308be01ca5 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -747,8 +747,6 @@ export const workflowRuns = { } /** - * Preview human input form content and placeholders - * * Get human input form preview for advanced chat workflow */ export const post4 = oc @@ -758,7 +756,6 @@ export const post4 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -768,15 +765,12 @@ export const post4 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview = { post: post4, } /** - * Submit human input form preview - * * Submit human input form preview for advanced chat workflow */ export const post5 = oc @@ -786,7 +780,6 @@ export const post5 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -796,7 +789,6 @@ export const post5 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run = { post: post5, @@ -850,8 +842,6 @@ export const humanInput = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node for advanced chat */ export const post7 = oc @@ -861,7 +851,6 @@ export const post7 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -871,7 +860,6 @@ export const post7 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run2 = { post: post7, @@ -890,8 +878,6 @@ export const iteration = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node for advanced chat */ export const post8 = oc @@ -901,7 +887,6 @@ export const post8 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -911,7 +896,6 @@ export const post8 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run3 = { post: post8, @@ -930,8 +914,6 @@ export const loop = { } /** - * Run draft workflow - * * Run draft workflow for advanced chat application */ export const post9 = oc @@ -941,7 +923,6 @@ export const post9 = oc method: 'POST', operationId: 'postAppsByAppIdAdvancedChatWorkflowsDraftRun', path: '/apps/{app_id}/advanced-chat/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -951,10 +932,7 @@ export const post9 = oc }), ) .output(zPostAppsByAppIdAdvancedChatWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run4 = { post: post9, @@ -2077,21 +2055,15 @@ export const conversationVariables = { } /** - * Convert basic mode of chatbot app to workflow mode - * * Convert application to workflow mode - * Convert expert mode of chatbot app to workflow mode - * Convert Completion App to Workflow App */ export const post23 = oc .route({ - description: - 'Convert application to workflow mode\nConvert expert mode of chatbot app to workflow mode\nConvert Completion App to Workflow App', + description: 'Convert application to workflow mode', inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdConvertToWorkflow', path: '/apps/{app_id}/convert-to-workflow', - summary: 'Convert basic mode of chatbot app to workflow mode', tags: ['console'], }) .input( @@ -2101,7 +2073,6 @@ export const post23 = oc }), ) .output(zPostAppsByAppIdConvertToWorkflowResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const convertToWorkflow = { post: post23, @@ -3180,8 +3151,6 @@ export const count3 = { } /** - * Stop workflow task - * * Stop running workflow task */ export const post41 = oc @@ -3191,12 +3160,10 @@ export const post41 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowRunsTasksByTaskIdStop', path: '/apps/{app_id}/workflow-runs/tasks/{task_id}/stop', - summary: 'Stop workflow task', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopPath })) .output(zPostAppsByAppIdWorkflowRunsTasksByTaskIdStopResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const stop3 = { post: post41, @@ -3818,8 +3785,6 @@ export const workflow = { } /** - * Get default block config - * * Get default block configuration by type */ export const get65 = oc @@ -3829,7 +3794,6 @@ export const get65 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockType', path: '/apps/{app_id}/workflows/default-workflow-block-configs/{block_type}', - summary: 'Get default block config', tags: ['console'], }) .input( @@ -3839,15 +3803,12 @@ export const get65 = oc }), ) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byBlockType = { get: get65, } /** - * Get default block config - * * Get default block configurations for workflow */ export const get66 = oc @@ -3857,12 +3818,10 @@ export const get66 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDefaultWorkflowBlockConfigs', path: '/apps/{app_id}/workflows/default-workflow-block-configs', - summary: 'Get default block config', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsPath })) .output(zGetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const defaultWorkflowBlockConfigs = { get: get66, @@ -3972,15 +3931,12 @@ export const post48 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftFeaturesResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const features = { post: post48, } /** - * Test human input delivery - * * Test human input delivery for workflow */ export const post49 = oc @@ -3990,7 +3946,6 @@ export const post49 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTest', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test', - summary: 'Test human input delivery', tags: ['console'], }) .input( @@ -4000,15 +3955,12 @@ export const post49 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const deliveryTest = { post: post49, } /** - * Preview human input form content and placeholders - * * Get human input form preview for workflow */ export const post50 = oc @@ -4018,7 +3970,6 @@ export const post50 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreview', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview', - summary: 'Preview human input form content and placeholders', tags: ['console'], }) .input( @@ -4028,15 +3979,12 @@ export const post50 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const preview4 = { post: post50, } /** - * Submit human input form preview - * * Submit human input form preview for workflow */ export const post51 = oc @@ -4046,7 +3994,6 @@ export const post51 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRun', path: '/apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run', - summary: 'Submit human input form preview', tags: ['console'], }) .input( @@ -4056,7 +4003,6 @@ export const post51 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run5 = { post: post51, @@ -4107,8 +4053,6 @@ export const humanInput2 = { } /** - * Run draft workflow iteration node - * * Run draft workflow iteration node */ export const post53 = oc @@ -4118,7 +4062,6 @@ export const post53 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run', - summary: 'Run draft workflow iteration node', tags: ['console'], }) .input( @@ -4128,7 +4071,6 @@ export const post53 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run6 = { post: post53, @@ -4147,8 +4089,6 @@ export const iteration2 = { } /** - * Run draft workflow loop node - * * Run draft workflow loop node */ export const post54 = oc @@ -4158,7 +4098,6 @@ export const post54 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run', - summary: 'Run draft workflow loop node', tags: ['console'], }) .input( @@ -4168,7 +4107,6 @@ export const post54 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run7 = { post: post54, @@ -4377,15 +4315,12 @@ export const get71 = oc }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunPath })) .output(zGetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const lastRun = { get: get71, } /** - * Run draft workflow node - * * Run draft workflow node */ export const post59 = oc @@ -4395,7 +4330,6 @@ export const post59 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/run', - summary: 'Run draft workflow node', tags: ['console'], }) .input( @@ -4405,15 +4339,12 @@ export const post59 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run8 = { post: post59, } /** - * Poll for trigger events and execute single node when event arrives - * * Poll for trigger events and execute single node when event arrives */ export const post60 = oc @@ -4423,12 +4354,10 @@ export const post60 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRun', path: '/apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run', - summary: 'Poll for trigger events and execute single node when event arrives', tags: ['console'], }) .input(z.object({ params: zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunPath })) .output(zPostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const run9 = { post: post60, @@ -4487,8 +4416,6 @@ export const nodes7 = { } /** - * Run draft workflow - * * Run draft workflow */ export const post61 = oc @@ -4498,7 +4425,6 @@ export const post61 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftRun', path: '/apps/{app_id}/workflows/draft/run', - summary: 'Run draft workflow', tags: ['console'], }) .input( @@ -4508,10 +4434,7 @@ export const post61 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run10 = { post: post61, @@ -4651,8 +4574,6 @@ export const systemVariables = { } /** - * Poll for trigger events and execute full workflow when event arrives - * * Poll for trigger events and execute full workflow when event arrives */ export const post62 = oc @@ -4662,7 +4583,6 @@ export const post62 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRun', path: '/apps/{app_id}/workflows/draft/trigger/run', - summary: 'Poll for trigger events and execute full workflow when event arrives', tags: ['console'], }) .input( @@ -4672,18 +4592,13 @@ export const post62 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const run11 = { post: post62, } /** - * Full workflow debug when the start node is a trigger - * * Full workflow debug when the start node is a trigger */ export const post63 = oc @@ -4693,7 +4608,6 @@ export const post63 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraftTriggerRunAll', path: '/apps/{app_id}/workflows/draft/trigger/run-all', - summary: 'Full workflow debug when the start node is a trigger', tags: ['console'], }) .input( @@ -4703,10 +4617,7 @@ export const post63 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftTriggerRunAllResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - rate_limit_error: { status: 429, data: zRateLimitError }, - }) + .errors({ rate_limit_error: { status: 429, data: zRateLimitError } }) export const runAll = { post: post63, @@ -4841,8 +4752,6 @@ export const variables2 = { } /** - * Get draft workflow - * * Get draft workflow for an application */ export const get80 = oc @@ -4852,19 +4761,13 @@ export const get80 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Get draft workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsDraftPath })) .output(zGetAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError }, - }) + .errors({ draft_workflow_not_exist: { status: 404, data: zDraftWorkflowNotExistError } }) /** - * Sync draft workflow - * * Sync draft workflow configuration */ export const post64 = oc @@ -4874,7 +4777,6 @@ export const post64 = oc method: 'POST', operationId: 'postAppsByAppIdWorkflowsDraft', path: '/apps/{app_id}/workflows/draft', - summary: 'Sync draft workflow', tags: ['console'], }) .input( @@ -4884,10 +4786,7 @@ export const post64 = oc }), ) .output(zPostAppsByAppIdWorkflowsDraftResponse) - .errors({ - app_not_found: { status: 404, data: zAppNotFoundError }, - draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - }) + .errors({ draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError } }) export const draft2 = { get: get80, @@ -4907,8 +4806,6 @@ export const draft2 = { } /** - * Get published workflow - * * Get published workflow for an application */ export const get81 = oc @@ -4918,23 +4815,17 @@ export const get81 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Get published workflow', tags: ['console'], }) .input(z.object({ params: zGetAppsByAppIdWorkflowsPublishPath })) .output(zGetAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) -/** - * Publish workflow - */ export const post65 = oc .route({ inputStructure: 'detailed', method: 'POST', operationId: 'postAppsByAppIdWorkflowsPublish', path: '/apps/{app_id}/workflows/publish', - summary: 'Publish workflow', tags: ['console'], }) .input( @@ -4944,7 +4835,6 @@ export const post65 = oc }), ) .output(zPostAppsByAppIdWorkflowsPublishResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const publish = { get: get81, @@ -5119,15 +5009,11 @@ export const post66 = oc }) .input(z.object({ params: zPostAppsByAppIdWorkflowsByWorkflowIdRestorePath })) .output(zPostAppsByAppIdWorkflowsByWorkflowIdRestoreResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const restore = { post: post66, } -/** - * Delete workflow - */ export const delete14 = oc .route({ inputStructure: 'detailed', @@ -5135,16 +5021,12 @@ export const delete14 = oc operationId: 'deleteAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete workflow', tags: ['console'], }) .input(z.object({ params: zDeleteAppsByAppIdWorkflowsByWorkflowIdPath })) .output(zDeleteAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) /** - * Update workflow attributes - * * Update workflow by ID */ export const patch3 = oc @@ -5154,7 +5036,6 @@ export const patch3 = oc method: 'PATCH', operationId: 'patchAppsByAppIdWorkflowsByWorkflowId', path: '/apps/{app_id}/workflows/{workflow_id}', - summary: 'Update workflow attributes', tags: ['console'], }) .input( @@ -5164,7 +5045,6 @@ export const patch3 = oc }), ) .output(zPatchAppsByAppIdWorkflowsByWorkflowIdResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const byWorkflowId = { delete: delete14, @@ -5173,8 +5053,6 @@ export const byWorkflowId = { } /** - * Get published workflows - * * Get all published workflows for an application */ export const get87 = oc @@ -5184,7 +5062,6 @@ export const get87 = oc method: 'GET', operationId: 'getAppsByAppIdWorkflows', path: '/apps/{app_id}/workflows', - summary: 'Get published workflows', tags: ['console'], }) .input( @@ -5194,7 +5071,6 @@ export const get87 = oc }), ) .output(zGetAppsByAppIdWorkflowsResponse) - .errors({ app_not_found: { status: 404, data: zAppNotFoundError } }) export const workflows3 = { get: get87, diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index b175d789433091..dc794fbcc29de8 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -5156,7 +5156,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdForm } export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormPreviewErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -5181,7 +5180,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdForm } export type PostAppsByAppIdAdvancedChatWorkflowsDraftHumanInputNodesByNodeIdFormRunErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -5235,7 +5233,7 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunDa export type PostAppsByAppIdAdvancedChatWorkflowsDraftIterationNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -5261,7 +5259,7 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunData = export type PostAppsByAppIdAdvancedChatWorkflowsDraftLoopNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -5287,7 +5285,6 @@ export type PostAppsByAppIdAdvancedChatWorkflowsDraftRunData = { export type PostAppsByAppIdAdvancedChatWorkflowsDraftRunErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse default: ErrorResponse } @@ -6566,7 +6563,6 @@ export type PostAppsByAppIdConvertToWorkflowData = { export type PostAppsByAppIdConvertToWorkflowErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -7806,7 +7802,7 @@ export type PostAppsByAppIdWorkflowRunsTasksByTaskIdStopData = { export type PostAppsByAppIdWorkflowRunsTasksByTaskIdStopErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8370,7 +8366,6 @@ export type GetAppsByAppIdWorkflowsData = { } export type GetAppsByAppIdWorkflowsErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8394,7 +8389,6 @@ export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsData = { } export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8421,7 +8415,7 @@ export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeData = } export type GetAppsByAppIdWorkflowsDefaultWorkflowBlockConfigsByBlockTypeErrors = { - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8445,7 +8439,7 @@ export type GetAppsByAppIdWorkflowsDraftData = { } export type GetAppsByAppIdWorkflowsDraftErrors = { - 404: AppNotFoundError | DraftWorkflowNotExistError | ErrorResponse + 404: DraftWorkflowNotExistError | ErrorResponse default: ErrorResponse } @@ -8471,7 +8465,6 @@ export type PostAppsByAppIdWorkflowsDraftData = { export type PostAppsByAppIdWorkflowsDraftErrors = { 400: ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 409: DraftWorkflowNotSyncError | ErrorResponse default: ErrorResponse } @@ -8590,7 +8583,6 @@ export type PostAppsByAppIdWorkflowsDraftFeaturesData = { } export type PostAppsByAppIdWorkflowsDraftFeaturesErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8615,7 +8607,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestData } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdDeliveryTestErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8640,7 +8631,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewData } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormPreviewErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8665,7 +8655,6 @@ export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunData = { } export type PostAppsByAppIdWorkflowsDraftHumanInputNodesByNodeIdFormRunErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -8716,7 +8705,7 @@ export type PostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftIterationNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8742,7 +8731,7 @@ export type PostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftLoopNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8964,7 +8953,7 @@ export type GetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunData = { export type GetAppsByAppIdWorkflowsDraftNodesByNodeIdLastRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -8990,7 +8979,7 @@ export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdRunData = { export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -9017,7 +9006,6 @@ export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunData = { export type PostAppsByAppIdWorkflowsDraftNodesByNodeIdTriggerRunErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 500: ErrorResponse default: ErrorResponse } @@ -9091,7 +9079,6 @@ export type PostAppsByAppIdWorkflowsDraftRunData = { export type PostAppsByAppIdWorkflowsDraftRunErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse default: ErrorResponse } @@ -9250,7 +9237,6 @@ export type PostAppsByAppIdWorkflowsDraftTriggerRunData = { export type PostAppsByAppIdWorkflowsDraftTriggerRunErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse 500: ErrorResponse default: ErrorResponse @@ -9278,7 +9264,6 @@ export type PostAppsByAppIdWorkflowsDraftTriggerRunAllData = { export type PostAppsByAppIdWorkflowsDraftTriggerRunAllErrors = { 400: WorkflowTriggerRunErrorResponse | ErrorResponse 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse 429: RateLimitError | ErrorResponse 500: ErrorResponse default: ErrorResponse @@ -9455,7 +9440,6 @@ export type GetAppsByAppIdWorkflowsPublishData = { } export type GetAppsByAppIdWorkflowsPublishErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9479,7 +9463,6 @@ export type PostAppsByAppIdWorkflowsPublishData = { } export type PostAppsByAppIdWorkflowsPublishErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9641,7 +9624,6 @@ export type DeleteAppsByAppIdWorkflowsByWorkflowIdData = { } export type DeleteAppsByAppIdWorkflowsByWorkflowIdErrors = { - 404: AppNotFoundError | ErrorResponse default: ErrorResponse } @@ -9667,7 +9649,7 @@ export type PatchAppsByAppIdWorkflowsByWorkflowIdData = { export type PatchAppsByAppIdWorkflowsByWorkflowIdErrors = { 403: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } @@ -9693,7 +9675,7 @@ export type PostAppsByAppIdWorkflowsByWorkflowIdRestoreData = { export type PostAppsByAppIdWorkflowsByWorkflowIdRestoreErrors = { 400: ErrorResponse - 404: AppNotFoundError | ErrorResponse + 404: ErrorResponse default: ErrorResponse } diff --git a/packages/contracts/generated/api/console/errors.gen.ts b/packages/contracts/generated/api/console/errors.gen.ts index fd47e4bbf7cf59..5f0e8fa93df185 100644 --- a/packages/contracts/generated/api/console/errors.gen.ts +++ b/packages/contracts/generated/api/console/errors.gen.ts @@ -5,7 +5,6 @@ export { delete11Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/sourc export { delete12Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}' } from './knowledge-fs/errors.gen.ts' export { delete12Errors as 'DELETE /workspaces/current/network-access-groups/{group_id}' } from './workspaces/errors.gen.ts' export { delete13Errors as 'DELETE /knowledge-fs/uploads/{upload_id}' } from './knowledge-fs/errors.gen.ts' -export { delete14Errors as 'DELETE /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { delete20Errors as 'DELETE /workspaces/current/skills/{skill_id}' } from './workspaces/errors.gen.ts' export { delete2Errors as 'DELETE /agent/{agent_id}/build-draft' } from './agent/errors.gen.ts' export { delete2Errors as 'DELETE /apps/{app_id}/agent/config/skills/{name}' } from './apps/errors.gen.ts' @@ -21,7 +20,6 @@ export { delete4Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/docume export { delete5Errors as 'DELETE /agent/{agent_id}/trace-config' } from './agent/errors.gen.ts' export { delete5Errors as 'DELETE /apps/{app_id}/chat-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete5Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { delete5Errors as 'DELETE /rag/pipelines/{pipeline_id}/workflows/{workflow_id}' } from './rag/errors.gen.ts' export { delete6Errors as 'DELETE /agent/{agent_id}' } from './agent/errors.gen.ts' export { delete6Errors as 'DELETE /apps/{app_id}/completion-conversations/{conversation_id}' } from './apps/errors.gen.ts' export { delete6Errors as 'DELETE /knowledge-fs/spaces/{control_space_id}/jobs/{job_id}' } from './knowledge-fs/errors.gen.ts' @@ -227,8 +225,6 @@ export { get61Errors as 'GET /apps/{app_id}/workflow/statistics/average-app-inte export { get62Errors as 'GET /apps/{app_id}/workflow/statistics/daily-conversations' } from './apps/errors.gen.ts' export { get63Errors as 'GET /apps/{app_id}/workflow/statistics/daily-terminals' } from './apps/errors.gen.ts' export { get64Errors as 'GET /apps/{app_id}/workflow/statistics/token-costs' } from './apps/errors.gen.ts' -export { get65Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs/{block_type}' } from './apps/errors.gen.ts' -export { get66Errors as 'GET /apps/{app_id}/workflows/default-workflow-block-configs' } from './apps/errors.gen.ts' export { get68Errors as 'GET /apps/{app_id}/workflows/draft/environment-variables' } from './apps/errors.gen.ts' export { get69Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/candidates' } from './apps/errors.gen.ts' export { get6Errors as 'GET /agent/{agent_id}/chat-messages' } from './agent/errors.gen.ts' @@ -238,7 +234,6 @@ export { get6Errors as 'GET /knowledge-fs/spaces/{control_space_id}/background-t export { get6Errors as 'GET /rag/pipelines/{pipeline_id}/exports' } from './rag/errors.gen.ts' export { get6Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/content' } from './trial-apps/errors.gen.ts' export { get70Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer' } from './apps/errors.gen.ts' -export { get71Errors as 'GET /apps/{app_id}/workflows/draft/nodes/{node_id}/last-run' } from './apps/errors.gen.ts' export { get73Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get74Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}/{output_name}/preview' } from './apps/errors.gen.ts' export { get75Errors as 'GET /apps/{app_id}/workflows/draft/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' @@ -253,13 +248,11 @@ export { get7Errors as 'GET /rag/pipelines/{pipeline_id}/workflow-runs/{run_id}/ export { get7Errors as 'GET /snippets/{snippet_id}/workflows/draft/environment-variables' } from './snippets/errors.gen.ts' export { get7Errors as 'GET /trial-apps/{app_id}/agent/config/skills/{name}/files/download' } from './trial-apps/errors.gen.ts' export { get80Errors as 'GET /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { get81Errors as 'GET /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' export { get82Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/events' } from './apps/errors.gen.ts' export { get83Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/{node_id}/{output_name}/preview' } from './apps/errors.gen.ts' export { get84Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs/{node_id}' } from './apps/errors.gen.ts' export { get85Errors as 'GET /apps/{app_id}/workflows/published/runs/{run_id}/node-outputs' } from './apps/errors.gen.ts' export { get86Errors as 'GET /apps/{app_id}/workflows/triggers/webhook' } from './apps/errors.gen.ts' -export { get87Errors as 'GET /apps/{app_id}/workflows' } from './apps/errors.gen.ts' export { get8Errors as 'GET /agent/{agent_id}/composer' } from './agent/errors.gen.ts' export { get8Errors as 'GET /apps/{app_id}/agent/config/files' } from './apps/errors.gen.ts' export { get8Errors as 'GET /installed-apps/{installed_app_id}' } from './installed-apps/errors.gen.ts' @@ -287,7 +280,6 @@ export { getErrors as 'GET /workflow-run-archives/downloads/{download_id}' } fro export { patch2Errors as 'PATCH /datasets/{dataset_id}/documents/status/{action}/batch' } from './datasets/errors.gen.ts' export { patch2Errors as 'PATCH /installed-apps/{installed_app_id}/conversations/{c_id}/unpin' } from './installed-apps/errors.gen.ts' export { patch2Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/golden-questions/{question_id}' } from './knowledge-fs/errors.gen.ts' -export { patch3Errors as 'PATCH /apps/{app_id}/workflows/{workflow_id}' } from './apps/errors.gen.ts' export { patch3Errors as 'PATCH /datasets/{dataset_id}/documents/{document_id}/processing/pause' } from './datasets/errors.gen.ts' export { patch3Errors as 'PATCH /installed-apps/{installed_app_id}' } from './installed-apps/errors.gen.ts' export { patch3Errors as 'PATCH /knowledge-fs/spaces/{control_space_id}/logical-documents/{document_id}' } from './knowledge-fs/errors.gen.ts' @@ -345,7 +337,6 @@ export { post15Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/draft' } f export { post16Errors as 'POST /agent/{agent_id}/publish' } from './agent/errors.gen.ts' export { post16Errors as 'POST /datasets/{dataset_id}/external-hit-testing' } from './datasets/errors.gen.ts' export { post16Errors as 'POST /knowledge-fs/spaces/{control_space_id}/queries/admission' } from './knowledge-fs/errors.gen.ts' -export { post16Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/publish' } from './rag/errors.gen.ts' export { post17Errors as 'POST /agent/{agent_id}/sandbox/files/download' } from './agent/errors.gen.ts' export { post17Errors as 'POST /datasets/{dataset_id}/hit-testing' } from './datasets/errors.gen.ts' export { post17Errors as 'POST /knowledge-fs/spaces/{control_space_id}/queries' } from './knowledge-fs/errors.gen.ts' @@ -360,7 +351,6 @@ export { post19Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/published/ export { post20Errors as 'POST /agent/{agent_id}/trace-config' } from './agent/errors.gen.ts' export { post20Errors as 'POST /apps/{app_id}/chat-messages/{task_id}/stop' } from './apps/errors.gen.ts' export { post20Errors as 'POST /knowledge-fs/spaces/{control_space_id}/research-tasks' } from './knowledge-fs/errors.gen.ts' -export { post20Errors as 'POST /rag/pipelines/{pipeline_id}/workflows/{workflow_id}/restore' } from './rag/errors.gen.ts' export { post21Errors as 'POST /agent/{agent_id}/trace-runs/{message_id}/retry' } from './agent/errors.gen.ts' export { post21Errors as 'POST /apps/{app_id}/completion-messages/{task_id}/stop' } from './apps/errors.gen.ts' export { post21Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections/{connection_id}/refresh' } from './knowledge-fs/errors.gen.ts' @@ -368,7 +358,6 @@ export { post22Errors as 'POST /agent/{agent_id}/versions/{version_id}/restore' export { post22Errors as 'POST /apps/{app_id}/completion-messages' } from './apps/errors.gen.ts' export { post22Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-connections' } from './knowledge-fs/errors.gen.ts' export { post23Errors as 'POST /agent' } from './agent/errors.gen.ts' -export { post23Errors as 'POST /apps/{app_id}/convert-to-workflow' } from './apps/errors.gen.ts' export { post23Errors as 'POST /datasets/{resource_id}/api-keys' } from './datasets/errors.gen.ts' export { post23Errors as 'POST /knowledge-fs/spaces/{control_space_id}/source-workflows/{run_id}/cancel' } from './knowledge-fs/errors.gen.ts' export { post24Errors as 'POST /datasets' } from './datasets/errors.gen.ts' @@ -427,7 +416,6 @@ export { post3Errors as 'POST /trial-apps/{app_id}/chat-messages' } from './tria export { post3Errors as 'POST /workflow-generate' } from './workflow-generate/errors.gen.ts' export { post40Errors as 'POST /apps/{app_id}/trigger-enable' } from './apps/errors.gen.ts' export { post40Errors as 'POST /knowledge-fs/spaces' } from './knowledge-fs/errors.gen.ts' -export { post41Errors as 'POST /apps/{app_id}/workflow-runs/tasks/{task_id}/stop' } from './apps/errors.gen.ts' export { post41Errors as 'POST /knowledge-fs/tasks/{task_id}/stream-capability' } from './knowledge-fs/errors.gen.ts' export { post41Errors as 'POST /workspaces/current/plugin/install/github' } from './workspaces/errors.gen.ts' export { post42Errors as 'POST /apps/{app_id}/workflow-runs/{workflow_run_id}/agent-nodes/{node_id}/sandbox/files/download' } from './apps/errors.gen.ts' @@ -440,41 +428,31 @@ export { post44Errors as 'POST /workspaces/current/plugin/list/installations/ids export { post45Errors as 'POST /apps/{app_id}/workflow/comments' } from './apps/errors.gen.ts' export { post45Errors as 'POST /workspaces/current/plugin/list/latest-versions' } from './workspaces/errors.gen.ts' export { post46Errors as 'POST /workspaces/current/plugin/parameters/dynamic-options-with-credentials' } from './workspaces/errors.gen.ts' -export { post48Errors as 'POST /apps/{app_id}/workflows/draft/features' } from './apps/errors.gen.ts' export { post48Errors as 'POST /workspaces/current/plugin/tasks/delete_all' } from './workspaces/errors.gen.ts' -export { post49Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/delivery-test' } from './apps/errors.gen.ts' export { post49Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete/{identifier}' } from './workspaces/errors.gen.ts' export { post4Errors as 'POST /account/change-email' } from './account/errors.gen.ts' export { post4Errors as 'POST /agent/{agent_id}/build-chat/finalize' } from './agent/errors.gen.ts' -export { post4Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' export { post4Errors as 'POST /datasets/indexing-estimate' } from './datasets/errors.gen.ts' export { post4Errors as 'POST /installed-apps/{installed_app_id}/completion-messages/{task_id}/stop' } from './installed-apps/errors.gen.ts' export { post4Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/cancel' } from './knowledge-fs/errors.gen.ts' export { post4Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './snippets/errors.gen.ts' export { post4Errors as 'POST /trial-apps/{app_id}/completion-messages' } from './trial-apps/errors.gen.ts' -export { post50Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/preview' } from './apps/errors.gen.ts' export { post50Errors as 'POST /workspaces/current/plugin/tasks/{task_id}/delete' } from './workspaces/errors.gen.ts' -export { post51Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post51Errors as 'POST /workspaces/current/plugin/uninstall' } from './workspaces/errors.gen.ts' export { post52Errors as 'POST /apps/{app_id}/workflows/draft/human-input/nodes/{node_id}/message-template/test' } from './apps/errors.gen.ts' export { post52Errors as 'POST /workspaces/current/plugin/upgrade/github' } from './workspaces/errors.gen.ts' -export { post53Errors as 'POST /apps/{app_id}/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post53Errors as 'POST /workspaces/current/plugin/upgrade/marketplace' } from './workspaces/errors.gen.ts' -export { post54Errors as 'POST /apps/{app_id}/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post55Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/copy-from-roster' } from './apps/errors.gen.ts' export { post55Errors as 'POST /workspaces/current/plugin/upload/github' } from './workspaces/errors.gen.ts' export { post56Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/impact' } from './apps/errors.gen.ts' export { post56Errors as 'POST /workspaces/current/plugin/upload/pkg' } from './workspaces/errors.gen.ts' export { post57Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/save-to-roster' } from './apps/errors.gen.ts' export { post58Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/agent-composer/validate' } from './apps/errors.gen.ts' -export { post59Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /agent/{agent_id}/build-draft/apply' } from './agent/errors.gen.ts' -export { post5Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/human-input/nodes/{node_id}/form/run' } from './apps/errors.gen.ts' export { post5Errors as 'POST /datasets/init' } from './datasets/errors.gen.ts' export { post5Errors as 'POST /installed-apps/{installed_app_id}/completion-messages' } from './installed-apps/errors.gen.ts' export { post5Errors as 'POST /knowledge-fs/spaces/{control_space_id}/background-tasks/{task_kind}/{task_id}/retry' } from './knowledge-fs/errors.gen.ts' export { post5Errors as 'POST /trial-apps/{app_id}/files/upload' } from './trial-apps/errors.gen.ts' -export { post60Errors as 'POST /apps/{app_id}/workflows/draft/nodes/{node_id}/trigger/run' } from './apps/errors.gen.ts' export { post61Errors as 'POST /apps/{app_id}/workflows/draft/run' } from './apps/errors.gen.ts' export { post61Errors as 'POST /workspaces/current/skills/files/upload' } from './workspaces/errors.gen.ts' export { post62Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run' } from './apps/errors.gen.ts' @@ -482,9 +460,7 @@ export { post62Errors as 'POST /workspaces/current/skills/import' } from './work export { post63Errors as 'POST /apps/{app_id}/workflows/draft/trigger/run-all' } from './apps/errors.gen.ts' export { post63Errors as 'POST /workspaces/current/skills/{skill_id}/assist/messages' } from './workspaces/errors.gen.ts' export { post64Errors as 'POST /apps/{app_id}/workflows/draft' } from './apps/errors.gen.ts' -export { post65Errors as 'POST /apps/{app_id}/workflows/publish' } from './apps/errors.gen.ts' export { post65Errors as 'POST /workspaces/current/skills/{skill_id}/files/check' } from './workspaces/errors.gen.ts' -export { post66Errors as 'POST /apps/{app_id}/workflows/{workflow_id}/restore' } from './apps/errors.gen.ts' export { post66Errors as 'POST /workspaces/current/skills/{skill_id}/publish' } from './workspaces/errors.gen.ts' export { post67Errors as 'POST /apps/{resource_id}/api-keys' } from './apps/errors.gen.ts' export { post67Errors as 'POST /workspaces/current/skills/{skill_id}/restore' } from './workspaces/errors.gen.ts' @@ -499,7 +475,6 @@ export { post6Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node export { post6Errors as 'POST /trial-apps/{app_id}/remote-files/upload' } from './trial-apps/errors.gen.ts' export { post7Errors as 'POST /account/education' } from './account/errors.gen.ts' export { post7Errors as 'POST /agent/{agent_id}/chat-messages/{task_id}/stop' } from './agent/errors.gen.ts' -export { post7Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/iteration/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post7Errors as 'POST /installed-apps/{installed_app_id}/messages/{message_id}/feedbacks' } from './installed-apps/errors.gen.ts' export { post7Errors as 'POST /knowledge-fs/spaces/{control_space_id}/documents' } from './knowledge-fs/errors.gen.ts' export { post7Errors as 'POST /rag/pipelines/{pipeline_id}/customized/publish' } from './rag/errors.gen.ts' @@ -507,7 +482,6 @@ export { post7Errors as 'POST /snippets/{snippet_id}/workflows/draft/nodes/{node export { post7Errors as 'POST /trial-apps/{app_id}/text-to-audio' } from './trial-apps/errors.gen.ts' export { post8Errors as 'POST /account/init' } from './account/errors.gen.ts' export { post8Errors as 'POST /agent/{agent_id}/composer/validate' } from './agent/errors.gen.ts' -export { post8Errors as 'POST /apps/{app_id}/advanced-chat/workflows/draft/loop/nodes/{node_id}/run' } from './apps/errors.gen.ts' export { post8Errors as 'POST /installed-apps/{installed_app_id}/saved-messages' } from './installed-apps/errors.gen.ts' export { post8Errors as 'POST /knowledge-fs/spaces/{control_space_id}/golden-questions/bulk-import' } from './knowledge-fs/errors.gen.ts' export { post8Errors as 'POST /rag/pipelines/{pipeline_id}/workflow-runs/tasks/{task_id}/stop' } from './rag/errors.gen.ts' diff --git a/packages/contracts/generated/api/console/rag/errors.gen.ts b/packages/contracts/generated/api/console/rag/errors.gen.ts index 501d339d7f2b47..6ba400dabbb13c 100644 --- a/packages/contracts/generated/api/console/rag/errors.gen.ts +++ b/packages/contracts/generated/api/console/rag/errors.gen.ts @@ -108,13 +108,10 @@ export const get20Errors = { export const post15Errors = { draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, } export const get21Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } -export const post16Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const post17Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const post18Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } @@ -128,10 +125,6 @@ export const post19Errors = { rate_limit_error: { status: 429, data: zRateLimitError }, } -export const post20Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - -export const delete5Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } - export const patch3Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } export const get24Errors = { pipeline_not_found: { status: 404, data: zPipelineNotFoundError } } diff --git a/packages/contracts/generated/api/console/rag/orpc.gen.ts b/packages/contracts/generated/api/console/rag/orpc.gen.ts index b266709612c4db..32abe153377753 100644 --- a/packages/contracts/generated/api/console/rag/orpc.gen.ts +++ b/packages/contracts/generated/api/console/rag/orpc.gen.ts @@ -1048,10 +1048,7 @@ export const post15 = oc }), ) .output(zPostRagPipelinesByPipelineIdWorkflowsDraftResponse) - .errors({ - draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError }, - pipeline_not_found: { status: 404, data: zPipelineNotFoundError }, - }) + .errors({ draft_workflow_not_sync: { status: 409, data: zDraftWorkflowNotSyncError } }) export const draft = { get: get20, @@ -1085,7 +1082,7 @@ export const get21 = oc .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) /** - * Publish workflow + * Publish a prepared Pipeline and its dataset settings atomically */ export const post16 = oc .route({ @@ -1093,12 +1090,11 @@ export const post16 = oc method: 'POST', operationId: 'postRagPipelinesByPipelineIdWorkflowsPublish', path: '/rag/pipelines/{pipeline_id}/workflows/publish', - summary: 'Publish workflow', + summary: 'Publish a prepared Pipeline and its dataset settings atomically', tags: ['console'], }) .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsPublishPath })) .output(zPostRagPipelinesByPipelineIdWorkflowsPublishResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) export const publish2 = { get: get21, @@ -1271,15 +1267,11 @@ export const post20 = oc }) .input(z.object({ params: zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestorePath })) .output(zPostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) export const restore = { post: post20, } -/** - * Delete a published workflow version that is not currently active on the pipeline - */ export const delete5 = oc .route({ inputStructure: 'detailed', @@ -1287,12 +1279,10 @@ export const delete5 = oc operationId: 'deleteRagPipelinesByPipelineIdWorkflowsByWorkflowId', path: '/rag/pipelines/{pipeline_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published workflow version that is not currently active on the pipeline', tags: ['console'], }) .input(z.object({ params: zDeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdPath })) .output(zDeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdResponse) - .errors({ pipeline_not_found: { status: 404, data: zPipelineNotFoundError } }) /** * Update workflow attributes diff --git a/packages/contracts/generated/api/console/rag/types.gen.ts b/packages/contracts/generated/api/console/rag/types.gen.ts index 11a7d9a559d684..ebaa560dc8f165 100644 --- a/packages/contracts/generated/api/console/rag/types.gen.ts +++ b/packages/contracts/generated/api/console/rag/types.gen.ts @@ -1549,7 +1549,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsDraftData = { } export type PostRagPipelinesByPipelineIdWorkflowsDraftErrors = { - 404: PipelineNotFoundError | ErrorResponse 409: DraftWorkflowNotSyncError | ErrorResponse default: ErrorResponse } @@ -2066,7 +2065,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsPublishData = { } export type PostRagPipelinesByPipelineIdWorkflowsPublishErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } @@ -2221,7 +2219,6 @@ export type DeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdData = { } export type DeleteRagPipelinesByPipelineIdWorkflowsByWorkflowIdErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } @@ -2273,7 +2270,6 @@ export type PostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreData = { } export type PostRagPipelinesByPipelineIdWorkflowsByWorkflowIdRestoreErrors = { - 404: PipelineNotFoundError | ErrorResponse default: ErrorResponse } diff --git a/packages/contracts/generated/api/console/snippets/orpc.gen.ts b/packages/contracts/generated/api/console/snippets/orpc.gen.ts index 635e7773a6928e..4d0b11051edc16 100644 --- a/packages/contracts/generated/api/console/snippets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/snippets/orpc.gen.ts @@ -918,7 +918,7 @@ export const restore = { } /** - * Delete a published snippet workflow version + * Delete a version and retire its unowned Agents through the shared use case * * Delete a published snippet workflow version */ @@ -930,7 +930,7 @@ export const delete4 = oc operationId: 'deleteSnippetsBySnippetIdWorkflowsByWorkflowId', path: '/snippets/{snippet_id}/workflows/{workflow_id}', successStatus: 204, - summary: 'Delete a published snippet workflow version', + summary: 'Delete a version and retire its unowned Agents through the shared use case', tags: ['console'], }) .input(z.object({ params: zDeleteSnippetsBySnippetIdWorkflowsByWorkflowIdPath }))